From f8a38378858861504a8739a31d6b3c04173908bd Mon Sep 17 00:00:00 2001 From: abhipray-chavan Date: Sun, 26 Apr 2026 11:33:04 +0000 Subject: [PATCH 1/7] Add TabM integration --- TEAM_MODULE_INVENTORY.md | 8 + tabm_integrated/FINAL_EXPERIMENT_REPORT.md | 56 +++++ tabm_integrated/__init__.py | 0 tabm_integrated/paper/bin/run_integrated.py | 198 ++++++++++++++++++ tabm_integrated/paper/data/.gitignore | 2 + tabm_integrated/paper/exp/.gitignore | 11 + .../paper/exp/final_integrated_audit.csv | 151 +++++++++++++ .../paper/exp/final_integrated_summary.csv | 51 +++++ .../all_four_combined-evaluation/0.toml | 35 ++++ .../all_four_combined-evaluation/0/DONE | 1 + .../0/report.json | 136 ++++++++++++ .../all_four_combined-evaluation/1.toml | 35 ++++ .../all_four_combined-evaluation/1/DONE | 1 + .../1/report.json | 106 ++++++++++ .../all_four_combined-evaluation/2.toml | 35 ++++ .../all_four_combined-evaluation/2/DONE | 1 + .../2/report.json | 118 +++++++++++ .../all_minus_cf_fisd-evaluation/0.toml | 34 +++ .../all_minus_cf_fisd-evaluation/0/DONE | 1 + .../0/report.json | 117 +++++++++++ .../all_minus_cf_fisd-evaluation/1.toml | 34 +++ .../all_minus_cf_fisd-evaluation/1/DONE | 1 + .../1/report.json | 105 ++++++++++ .../all_minus_cf_fisd-evaluation/2.toml | 34 +++ .../all_minus_cf_fisd-evaluation/2/DONE | 1 + .../2/report.json | 111 ++++++++++ .../all_minus_esam-evaluation/0.toml | 33 +++ .../all_minus_esam-evaluation/0/DONE | 1 + .../all_minus_esam-evaluation/0/report.json | 182 ++++++++++++++++ .../all_minus_esam-evaluation/1.toml | 33 +++ .../all_minus_esam-evaluation/1/DONE | 1 + .../all_minus_esam-evaluation/1/report.json | 134 ++++++++++++ .../all_minus_esam-evaluation/2.toml | 33 +++ .../all_minus_esam-evaluation/2/DONE | 1 + .../all_minus_esam-evaluation/2/report.json | 146 +++++++++++++ .../all_minus_mfb-evaluation/0.toml | 33 +++ .../all_minus_mfb-evaluation/0/DONE | 1 + .../all_minus_mfb-evaluation/0/report.json | 122 +++++++++++ .../all_minus_mfb-evaluation/1.toml | 33 +++ .../all_minus_mfb-evaluation/1/DONE | 1 + .../all_minus_mfb-evaluation/1/report.json | 122 +++++++++++ .../all_minus_mfb-evaluation/2.toml | 33 +++ .../all_minus_mfb-evaluation/2/DONE | 1 + .../all_minus_mfb-evaluation/2/report.json | 110 ++++++++++ .../all_minus_rla-evaluation/0.toml | 32 +++ .../all_minus_rla-evaluation/0/DONE | 1 + .../all_minus_rla-evaluation/0/report.json | 139 ++++++++++++ .../all_minus_rla-evaluation/1.toml | 32 +++ .../all_minus_rla-evaluation/1/DONE | 1 + .../all_minus_rla-evaluation/1/report.json | 103 +++++++++ .../all_minus_rla-evaluation/2.toml | 32 +++ .../all_minus_rla-evaluation/2/DONE | 1 + .../all_minus_rla-evaluation/2/report.json | 157 ++++++++++++++ .../baseline_plr-evaluation/0.toml | 27 +++ .../baseline_plr-evaluation/0/DONE | 1 + .../baseline_plr-evaluation/0/report.json | 116 ++++++++++ .../baseline_plr-evaluation/1.toml | 27 +++ .../baseline_plr-evaluation/1/DONE | 1 + .../baseline_plr-evaluation/1/report.json | 116 ++++++++++ .../baseline_plr-evaluation/2.toml | 27 +++ .../baseline_plr-evaluation/2/DONE | 1 + .../baseline_plr-evaluation/2/report.json | 158 ++++++++++++++ .../cf_fisd_only-evaluation/0.toml | 28 +++ .../cf_fisd_only-evaluation/0/DONE | 1 + .../cf_fisd_only-evaluation/0/report.json | 99 +++++++++ .../cf_fisd_only-evaluation/1.toml | 28 +++ .../cf_fisd_only-evaluation/1/DONE | 1 + .../cf_fisd_only-evaluation/1/report.json | 117 +++++++++++ .../cf_fisd_only-evaluation/2.toml | 28 +++ .../cf_fisd_only-evaluation/2/DONE | 1 + .../cf_fisd_only-evaluation/2/report.json | 159 ++++++++++++++ .../cooking-time/esam_only-evaluation/0.toml | 29 +++ .../cooking-time/esam_only-evaluation/0/DONE | 1 + .../esam_only-evaluation/0/report.json | 118 +++++++++++ .../cooking-time/esam_only-evaluation/1.toml | 29 +++ .../cooking-time/esam_only-evaluation/1/DONE | 1 + .../esam_only-evaluation/1/report.json | 118 +++++++++++ .../cooking-time/esam_only-evaluation/2.toml | 29 +++ .../cooking-time/esam_only-evaluation/2/DONE | 1 + .../esam_only-evaluation/2/report.json | 160 ++++++++++++++ .../cooking-time/mfb_only-evaluation/0.toml | 29 +++ .../cooking-time/mfb_only-evaluation/0/DONE | 1 + .../mfb_only-evaluation/0/report.json | 130 ++++++++++++ .../cooking-time/mfb_only-evaluation/1.toml | 29 +++ .../cooking-time/mfb_only-evaluation/1/DONE | 1 + .../mfb_only-evaluation/1/report.json | 142 +++++++++++++ .../cooking-time/mfb_only-evaluation/2.toml | 29 +++ .../cooking-time/mfb_only-evaluation/2/DONE | 1 + .../mfb_only-evaluation/2/report.json | 142 +++++++++++++ .../cooking-time/rla_only-evaluation/0.toml | 30 +++ .../cooking-time/rla_only-evaluation/0/DONE | 1 + .../rla_only-evaluation/0/report.json | 119 +++++++++++ .../cooking-time/rla_only-evaluation/1.toml | 30 +++ .../cooking-time/rla_only-evaluation/1/DONE | 1 + .../rla_only-evaluation/1/report.json | 119 +++++++++++ .../cooking-time/rla_only-evaluation/2.toml | 30 +++ .../cooking-time/rla_only-evaluation/2/DONE | 1 + .../rla_only-evaluation/2/report.json | 107 ++++++++++ .../all_four_combined-evaluation/0.toml | 35 ++++ .../all_four_combined-evaluation/0/DONE | 1 + .../0/report.json | 106 ++++++++++ .../all_four_combined-evaluation/1.toml | 35 ++++ .../all_four_combined-evaluation/1/DONE | 1 + .../1/report.json | 112 ++++++++++ .../all_four_combined-evaluation/2.toml | 35 ++++ .../all_four_combined-evaluation/2/DONE | 1 + .../2/report.json | 106 ++++++++++ .../all_minus_cf_fisd-evaluation/0.toml | 34 +++ .../all_minus_cf_fisd-evaluation/0/DONE | 1 + .../0/report.json | 105 ++++++++++ .../all_minus_cf_fisd-evaluation/1.toml | 34 +++ .../all_minus_cf_fisd-evaluation/1/DONE | 1 + .../1/report.json | 111 ++++++++++ .../all_minus_cf_fisd-evaluation/2.toml | 34 +++ .../all_minus_cf_fisd-evaluation/2/DONE | 1 + .../2/report.json | 111 ++++++++++ .../all_minus_esam-evaluation/0.toml | 33 +++ .../all_minus_esam-evaluation/0/DONE | 1 + .../all_minus_esam-evaluation/0/report.json | 104 +++++++++ .../all_minus_esam-evaluation/1.toml | 33 +++ .../all_minus_esam-evaluation/1/DONE | 1 + .../all_minus_esam-evaluation/1/report.json | 122 +++++++++++ .../all_minus_esam-evaluation/2.toml | 33 +++ .../all_minus_esam-evaluation/2/DONE | 1 + .../all_minus_esam-evaluation/2/report.json | 104 +++++++++ .../all_minus_mfb-evaluation/0.toml | 33 +++ .../all_minus_mfb-evaluation/0/DONE | 1 + .../all_minus_mfb-evaluation/0/report.json | 128 +++++++++++ .../all_minus_mfb-evaluation/1.toml | 33 +++ .../all_minus_mfb-evaluation/1/DONE | 1 + .../all_minus_mfb-evaluation/1/report.json | 116 ++++++++++ .../all_minus_mfb-evaluation/2.toml | 33 +++ .../all_minus_mfb-evaluation/2/DONE | 1 + .../all_minus_mfb-evaluation/2/report.json | 104 +++++++++ .../all_minus_rla-evaluation/0.toml | 32 +++ .../all_minus_rla-evaluation/0/DONE | 1 + .../all_minus_rla-evaluation/0/report.json | 103 +++++++++ .../all_minus_rla-evaluation/1.toml | 32 +++ .../all_minus_rla-evaluation/1/DONE | 1 + .../all_minus_rla-evaluation/1/report.json | 109 ++++++++++ .../all_minus_rla-evaluation/2.toml | 32 +++ .../all_minus_rla-evaluation/2/DONE | 1 + .../all_minus_rla-evaluation/2/report.json | 103 +++++++++ .../baseline_plr-evaluation/0.toml | 27 +++ .../baseline_plr-evaluation/0/DONE | 1 + .../baseline_plr-evaluation/0/report.json | 98 +++++++++ .../baseline_plr-evaluation/1.toml | 27 +++ .../baseline_plr-evaluation/1/DONE | 1 + .../baseline_plr-evaluation/1/report.json | 98 +++++++++ .../baseline_plr-evaluation/2.toml | 27 +++ .../baseline_plr-evaluation/2/DONE | 1 + .../baseline_plr-evaluation/2/report.json | 122 +++++++++++ .../cf_fisd_only-evaluation/0.toml | 28 +++ .../cf_fisd_only-evaluation/0/DONE | 1 + .../cf_fisd_only-evaluation/0/report.json | 99 +++++++++ .../cf_fisd_only-evaluation/1.toml | 28 +++ .../cf_fisd_only-evaluation/1/DONE | 1 + .../cf_fisd_only-evaluation/1/report.json | 105 ++++++++++ .../cf_fisd_only-evaluation/2.toml | 28 +++ .../cf_fisd_only-evaluation/2/DONE | 1 + .../cf_fisd_only-evaluation/2/report.json | 123 +++++++++++ .../delivery-eta/esam_only-evaluation/0.toml | 29 +++ .../delivery-eta/esam_only-evaluation/0/DONE | 1 + .../esam_only-evaluation/0/report.json | 100 +++++++++ .../delivery-eta/esam_only-evaluation/1.toml | 29 +++ .../delivery-eta/esam_only-evaluation/1/DONE | 1 + .../esam_only-evaluation/1/report.json | 100 +++++++++ .../delivery-eta/esam_only-evaluation/2.toml | 29 +++ .../delivery-eta/esam_only-evaluation/2/DONE | 1 + .../esam_only-evaluation/2/report.json | 124 +++++++++++ .../delivery-eta/mfb_only-evaluation/0.toml | 29 +++ .../delivery-eta/mfb_only-evaluation/0/DONE | 1 + .../mfb_only-evaluation/0/report.json | 100 +++++++++ .../delivery-eta/mfb_only-evaluation/1.toml | 29 +++ .../delivery-eta/mfb_only-evaluation/1/DONE | 1 + .../mfb_only-evaluation/1/report.json | 124 +++++++++++ .../delivery-eta/mfb_only-evaluation/2.toml | 29 +++ .../delivery-eta/mfb_only-evaluation/2/DONE | 1 + .../mfb_only-evaluation/2/report.json | 136 ++++++++++++ .../delivery-eta/rla_only-evaluation/0.toml | 30 +++ .../delivery-eta/rla_only-evaluation/0/DONE | 1 + .../rla_only-evaluation/0/report.json | 161 ++++++++++++++ .../delivery-eta/rla_only-evaluation/1.toml | 30 +++ .../delivery-eta/rla_only-evaluation/1/DONE | 1 + .../rla_only-evaluation/1/report.json | 113 ++++++++++ .../delivery-eta/rla_only-evaluation/2.toml | 30 +++ .../delivery-eta/rla_only-evaluation/2/DONE | 1 + .../rla_only-evaluation/2/report.json | 101 +++++++++ .../all_four_combined-evaluation/0.toml | 35 ++++ .../all_four_combined-evaluation/0/DONE | 1 + .../0/report.json | 184 ++++++++++++++++ .../all_four_combined-evaluation/1.toml | 35 ++++ .../all_four_combined-evaluation/1/DONE | 1 + .../1/report.json | 184 ++++++++++++++++ .../all_four_combined-evaluation/2.toml | 35 ++++ .../all_four_combined-evaluation/2/DONE | 1 + .../2/report.json | 112 ++++++++++ .../all_minus_cf_fisd-evaluation/0.toml | 34 +++ .../all_minus_cf_fisd-evaluation/0/DONE | 1 + .../0/report.json | 105 ++++++++++ .../all_minus_cf_fisd-evaluation/1.toml | 34 +++ .../all_minus_cf_fisd-evaluation/1/DONE | 1 + .../1/report.json | 105 ++++++++++ .../all_minus_cf_fisd-evaluation/2.toml | 34 +++ .../all_minus_cf_fisd-evaluation/2/DONE | 1 + .../2/report.json | 183 ++++++++++++++++ .../all_minus_esam-evaluation/0.toml | 33 +++ .../all_minus_esam-evaluation/0/DONE | 1 + .../all_minus_esam-evaluation/0/report.json | 140 +++++++++++++ .../all_minus_esam-evaluation/1.toml | 33 +++ .../all_minus_esam-evaluation/1/DONE | 1 + .../all_minus_esam-evaluation/1/report.json | 110 ++++++++++ .../all_minus_esam-evaluation/2.toml | 33 +++ .../all_minus_esam-evaluation/2/DONE | 1 + .../all_minus_esam-evaluation/2/report.json | 110 ++++++++++ .../all_minus_mfb-evaluation/0.toml | 33 +++ .../all_minus_mfb-evaluation/0/DONE | 1 + .../all_minus_mfb-evaluation/0/report.json | 140 +++++++++++++ .../all_minus_mfb-evaluation/1.toml | 33 +++ .../all_minus_mfb-evaluation/1/DONE | 1 + .../all_minus_mfb-evaluation/1/report.json | 146 +++++++++++++ .../all_minus_mfb-evaluation/2.toml | 33 +++ .../all_minus_mfb-evaluation/2/DONE | 1 + .../all_minus_mfb-evaluation/2/report.json | 122 +++++++++++ .../all_minus_rla-evaluation/0.toml | 32 +++ .../all_minus_rla-evaluation/0/DONE | 1 + .../all_minus_rla-evaluation/0/report.json | 103 +++++++++ .../all_minus_rla-evaluation/1.toml | 32 +++ .../all_minus_rla-evaluation/1/DONE | 1 + .../all_minus_rla-evaluation/1/report.json | 133 ++++++++++++ .../all_minus_rla-evaluation/2.toml | 32 +++ .../all_minus_rla-evaluation/2/DONE | 1 + .../all_minus_rla-evaluation/2/report.json | 109 ++++++++++ .../baseline_plr-evaluation/0.toml | 27 +++ .../baseline_plr-evaluation/0/DONE | 1 + .../baseline_plr-evaluation/0/report.json | 176 ++++++++++++++++ .../baseline_plr-evaluation/1.toml | 27 +++ .../baseline_plr-evaluation/1/DONE | 1 + .../baseline_plr-evaluation/1/report.json | 176 ++++++++++++++++ .../baseline_plr-evaluation/2.toml | 27 +++ .../baseline_plr-evaluation/2/DONE | 1 + .../baseline_plr-evaluation/2/report.json | 176 ++++++++++++++++ .../cf_fisd_only-evaluation/0.toml | 28 +++ .../cf_fisd_only-evaluation/0/DONE | 1 + .../cf_fisd_only-evaluation/0/report.json | 141 +++++++++++++ .../cf_fisd_only-evaluation/1.toml | 28 +++ .../cf_fisd_only-evaluation/1/DONE | 1 + .../cf_fisd_only-evaluation/1/report.json | 135 ++++++++++++ .../cf_fisd_only-evaluation/2.toml | 28 +++ .../cf_fisd_only-evaluation/2/DONE | 1 + .../cf_fisd_only-evaluation/2/report.json | 135 ++++++++++++ .../ecom-offers/esam_only-evaluation/0.toml | 29 +++ .../ecom-offers/esam_only-evaluation/0/DONE | 1 + .../esam_only-evaluation/0/report.json | 178 ++++++++++++++++ .../ecom-offers/esam_only-evaluation/1.toml | 29 +++ .../ecom-offers/esam_only-evaluation/1/DONE | 1 + .../esam_only-evaluation/1/report.json | 178 ++++++++++++++++ .../ecom-offers/esam_only-evaluation/2.toml | 29 +++ .../ecom-offers/esam_only-evaluation/2/DONE | 1 + .../esam_only-evaluation/2/report.json | 178 ++++++++++++++++ .../ecom-offers/mfb_only-evaluation/0.toml | 29 +++ .../ecom-offers/mfb_only-evaluation/0/DONE | 1 + .../mfb_only-evaluation/0/report.json | 100 +++++++++ .../ecom-offers/mfb_only-evaluation/1.toml | 29 +++ .../ecom-offers/mfb_only-evaluation/1/DONE | 1 + .../mfb_only-evaluation/1/report.json | 100 +++++++++ .../ecom-offers/mfb_only-evaluation/2.toml | 29 +++ .../ecom-offers/mfb_only-evaluation/2/DONE | 1 + .../mfb_only-evaluation/2/report.json | 178 ++++++++++++++++ .../ecom-offers/rla_only-evaluation/0.toml | 30 +++ .../ecom-offers/rla_only-evaluation/0/DONE | 1 + .../rla_only-evaluation/0/report.json | 179 ++++++++++++++++ .../ecom-offers/rla_only-evaluation/1.toml | 30 +++ .../ecom-offers/rla_only-evaluation/1/DONE | 1 + .../rla_only-evaluation/1/report.json | 179 ++++++++++++++++ .../ecom-offers/rla_only-evaluation/2.toml | 30 +++ .../ecom-offers/rla_only-evaluation/2/DONE | 1 + .../rla_only-evaluation/2/report.json | 179 ++++++++++++++++ .../all_four_combined-evaluation/0.toml | 35 ++++ .../all_four_combined-evaluation/0/DONE | 1 + .../0/report.json | 184 ++++++++++++++++ .../all_four_combined-evaluation/1.toml | 35 ++++ .../all_four_combined-evaluation/1/DONE | 1 + .../1/report.json | 184 ++++++++++++++++ .../all_four_combined-evaluation/2.toml | 35 ++++ .../all_four_combined-evaluation/2/DONE | 1 + .../2/report.json | 184 ++++++++++++++++ .../all_minus_cf_fisd-evaluation/0.toml | 34 +++ .../all_minus_cf_fisd-evaluation/0/DONE | 1 + .../0/report.json | 183 ++++++++++++++++ .../all_minus_cf_fisd-evaluation/1.toml | 34 +++ .../all_minus_cf_fisd-evaluation/1/DONE | 1 + .../1/report.json | 183 ++++++++++++++++ .../all_minus_cf_fisd-evaluation/2.toml | 34 +++ .../all_minus_cf_fisd-evaluation/2/DONE | 1 + .../2/report.json | 183 ++++++++++++++++ .../all_minus_esam-evaluation/0.toml | 33 +++ .../all_minus_esam-evaluation/0/DONE | 1 + .../all_minus_esam-evaluation/0/report.json | 182 ++++++++++++++++ .../all_minus_esam-evaluation/1.toml | 33 +++ .../all_minus_esam-evaluation/1/DONE | 1 + .../all_minus_esam-evaluation/1/report.json | 182 ++++++++++++++++ .../all_minus_esam-evaluation/2.toml | 33 +++ .../all_minus_esam-evaluation/2/DONE | 1 + .../all_minus_esam-evaluation/2/report.json | 182 ++++++++++++++++ .../all_minus_mfb-evaluation/0.toml | 33 +++ .../all_minus_mfb-evaluation/0/DONE | 1 + .../all_minus_mfb-evaluation/0/report.json | 182 ++++++++++++++++ .../all_minus_mfb-evaluation/1.toml | 33 +++ .../all_minus_mfb-evaluation/1/DONE | 1 + .../all_minus_mfb-evaluation/1/report.json | 182 ++++++++++++++++ .../all_minus_mfb-evaluation/2.toml | 33 +++ .../all_minus_mfb-evaluation/2/DONE | 1 + .../all_minus_mfb-evaluation/2/report.json | 182 ++++++++++++++++ .../all_minus_rla-evaluation/0.toml | 32 +++ .../all_minus_rla-evaluation/0/DONE | 1 + .../all_minus_rla-evaluation/0/report.json | 181 ++++++++++++++++ .../all_minus_rla-evaluation/1.toml | 32 +++ .../all_minus_rla-evaluation/1/DONE | 1 + .../all_minus_rla-evaluation/1/report.json | 181 ++++++++++++++++ .../all_minus_rla-evaluation/2.toml | 32 +++ .../all_minus_rla-evaluation/2/DONE | 1 + .../all_minus_rla-evaluation/2/report.json | 181 ++++++++++++++++ .../baseline_plr-evaluation/0.toml | 27 +++ .../baseline_plr-evaluation/0/DONE | 1 + .../baseline_plr-evaluation/0/report.json | 176 ++++++++++++++++ .../baseline_plr-evaluation/1.toml | 27 +++ .../baseline_plr-evaluation/1/DONE | 1 + .../baseline_plr-evaluation/1/report.json | 176 ++++++++++++++++ .../baseline_plr-evaluation/2.toml | 27 +++ .../baseline_plr-evaluation/2/DONE | 1 + .../baseline_plr-evaluation/2/report.json | 176 ++++++++++++++++ .../cf_fisd_only-evaluation/0.toml | 28 +++ .../cf_fisd_only-evaluation/0/DONE | 1 + .../cf_fisd_only-evaluation/0/report.json | 177 ++++++++++++++++ .../cf_fisd_only-evaluation/1.toml | 28 +++ .../cf_fisd_only-evaluation/1/DONE | 1 + .../cf_fisd_only-evaluation/1/report.json | 177 ++++++++++++++++ .../cf_fisd_only-evaluation/2.toml | 28 +++ .../cf_fisd_only-evaluation/2/DONE | 1 + .../cf_fisd_only-evaluation/2/report.json | 177 ++++++++++++++++ .../esam_only-evaluation/0.toml | 29 +++ .../esam_only-evaluation/0/DONE | 1 + .../esam_only-evaluation/0/report.json | 178 ++++++++++++++++ .../esam_only-evaluation/1.toml | 29 +++ .../esam_only-evaluation/1/DONE | 1 + .../esam_only-evaluation/1/report.json | 178 ++++++++++++++++ .../esam_only-evaluation/2.toml | 29 +++ .../esam_only-evaluation/2/DONE | 1 + .../esam_only-evaluation/2/report.json | 178 ++++++++++++++++ .../mfb_only-evaluation/0.toml | 29 +++ .../mfb_only-evaluation/0/DONE | 1 + .../mfb_only-evaluation/0/report.json | 178 ++++++++++++++++ .../mfb_only-evaluation/1.toml | 29 +++ .../mfb_only-evaluation/1/DONE | 1 + .../mfb_only-evaluation/1/report.json | 178 ++++++++++++++++ .../mfb_only-evaluation/2.toml | 29 +++ .../mfb_only-evaluation/2/DONE | 1 + .../mfb_only-evaluation/2/report.json | 178 ++++++++++++++++ .../rla_only-evaluation/0.toml | 30 +++ .../rla_only-evaluation/0/DONE | 1 + .../rla_only-evaluation/0/report.json | 179 ++++++++++++++++ .../rla_only-evaluation/1.toml | 30 +++ .../rla_only-evaluation/1/DONE | 1 + .../rla_only-evaluation/1/report.json | 179 ++++++++++++++++ .../rla_only-evaluation/2.toml | 30 +++ .../rla_only-evaluation/2/DONE | 1 + .../rla_only-evaluation/2/report.json | 179 ++++++++++++++++ .../all_four_combined-evaluation/0.toml | 35 ++++ .../all_four_combined-evaluation/0/DONE | 1 + .../0/report.json | 106 ++++++++++ .../all_four_combined-evaluation/1.toml | 35 ++++ .../all_four_combined-evaluation/1/DONE | 1 + .../1/report.json | 106 ++++++++++ .../all_four_combined-evaluation/2.toml | 35 ++++ .../all_four_combined-evaluation/2/DONE | 1 + .../2/report.json | 106 ++++++++++ .../all_minus_cf_fisd-evaluation/0.toml | 34 +++ .../all_minus_cf_fisd-evaluation/0/DONE | 1 + .../0/report.json | 117 +++++++++++ .../all_minus_cf_fisd-evaluation/1.toml | 34 +++ .../all_minus_cf_fisd-evaluation/1/DONE | 1 + .../1/report.json | 105 ++++++++++ .../all_minus_cf_fisd-evaluation/2.toml | 34 +++ .../all_minus_cf_fisd-evaluation/2/DONE | 1 + .../2/report.json | 105 ++++++++++ .../all_minus_esam-evaluation/0.toml | 33 +++ .../all_minus_esam-evaluation/0/DONE | 1 + .../all_minus_esam-evaluation/0/report.json | 104 +++++++++ .../all_minus_esam-evaluation/1.toml | 33 +++ .../all_minus_esam-evaluation/1/DONE | 1 + .../all_minus_esam-evaluation/1/report.json | 104 +++++++++ .../all_minus_esam-evaluation/2.toml | 33 +++ .../all_minus_esam-evaluation/2/DONE | 1 + .../all_minus_esam-evaluation/2/report.json | 104 +++++++++ .../all_minus_mfb-evaluation/0.toml | 33 +++ .../all_minus_mfb-evaluation/0/DONE | 1 + .../all_minus_mfb-evaluation/0/report.json | 104 +++++++++ .../all_minus_mfb-evaluation/1.toml | 33 +++ .../all_minus_mfb-evaluation/1/DONE | 1 + .../all_minus_mfb-evaluation/1/report.json | 104 +++++++++ .../all_minus_mfb-evaluation/2.toml | 33 +++ .../all_minus_mfb-evaluation/2/DONE | 1 + .../all_minus_mfb-evaluation/2/report.json | 104 +++++++++ .../all_minus_rla-evaluation/0.toml | 32 +++ .../all_minus_rla-evaluation/0/DONE | 1 + .../all_minus_rla-evaluation/0/report.json | 127 +++++++++++ .../all_minus_rla-evaluation/1.toml | 32 +++ .../all_minus_rla-evaluation/1/DONE | 1 + .../all_minus_rla-evaluation/1/report.json | 133 ++++++++++++ .../all_minus_rla-evaluation/2.toml | 32 +++ .../all_minus_rla-evaluation/2/DONE | 1 + .../all_minus_rla-evaluation/2/report.json | 103 +++++++++ .../baseline_plr-evaluation/0.toml | 27 +++ .../baseline_plr-evaluation/0/DONE | 1 + .../baseline_plr-evaluation/0/report.json | 128 +++++++++++ .../baseline_plr-evaluation/1.toml | 27 +++ .../baseline_plr-evaluation/1/DONE | 1 + .../baseline_plr-evaluation/1/report.json | 104 +++++++++ .../baseline_plr-evaluation/2.toml | 27 +++ .../baseline_plr-evaluation/2/DONE | 1 + .../baseline_plr-evaluation/2/report.json | 176 ++++++++++++++++ .../cf_fisd_only-evaluation/0.toml | 28 +++ .../cf_fisd_only-evaluation/0/DONE | 1 + .../cf_fisd_only-evaluation/0/report.json | 129 ++++++++++++ .../cf_fisd_only-evaluation/1.toml | 28 +++ .../cf_fisd_only-evaluation/1/DONE | 1 + .../cf_fisd_only-evaluation/1/report.json | 105 ++++++++++ .../cf_fisd_only-evaluation/2.toml | 28 +++ .../cf_fisd_only-evaluation/2/DONE | 1 + .../cf_fisd_only-evaluation/2/report.json | 99 +++++++++ .../esam_only-evaluation/0.toml | 29 +++ .../esam_only-evaluation/0/DONE | 1 + .../esam_only-evaluation/0/report.json | 130 ++++++++++++ .../esam_only-evaluation/1.toml | 29 +++ .../esam_only-evaluation/1/DONE | 1 + .../esam_only-evaluation/1/report.json | 106 ++++++++++ .../esam_only-evaluation/2.toml | 29 +++ .../esam_only-evaluation/2/DONE | 1 + .../esam_only-evaluation/2/report.json | 178 ++++++++++++++++ .../mfb_only-evaluation/0.toml | 29 +++ .../mfb_only-evaluation/0/DONE | 1 + .../mfb_only-evaluation/0/report.json | 118 +++++++++++ .../mfb_only-evaluation/1.toml | 29 +++ .../mfb_only-evaluation/1/DONE | 1 + .../mfb_only-evaluation/1/report.json | 106 ++++++++++ .../mfb_only-evaluation/2.toml | 29 +++ .../mfb_only-evaluation/2/DONE | 1 + .../mfb_only-evaluation/2/report.json | 124 +++++++++++ .../rla_only-evaluation/0.toml | 30 +++ .../rla_only-evaluation/0/DONE | 1 + .../rla_only-evaluation/0/report.json | 113 ++++++++++ .../rla_only-evaluation/1.toml | 30 +++ .../rla_only-evaluation/1/DONE | 1 + .../rla_only-evaluation/1/report.json | 101 +++++++++ .../rla_only-evaluation/2.toml | 30 +++ .../rla_only-evaluation/2/DONE | 1 + .../rla_only-evaluation/2/report.json | 101 +++++++++ tabm_integrated/pbs/run_integrated_16gpu.pbs | 4 + .../tools/aggregate_integrated_results.py | 45 ++++ .../tools/generate_integrated_configs.py | 27 +++ .../tools/run_integrated_matrix.sh | 79 +++++++ 462 files changed, 25832 insertions(+) create mode 100644 TEAM_MODULE_INVENTORY.md create mode 100644 tabm_integrated/FINAL_EXPERIMENT_REPORT.md create mode 100644 tabm_integrated/__init__.py create mode 100644 tabm_integrated/paper/bin/run_integrated.py create mode 100644 tabm_integrated/paper/data/.gitignore create mode 100644 tabm_integrated/paper/exp/.gitignore create mode 100644 tabm_integrated/paper/exp/final_integrated_audit.csv create mode 100644 tabm_integrated/paper/exp/final_integrated_summary.csv create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2/report.json create mode 100644 tabm_integrated/pbs/run_integrated_16gpu.pbs create mode 100644 tabm_integrated/tools/aggregate_integrated_results.py create mode 100644 tabm_integrated/tools/generate_integrated_configs.py create mode 100755 tabm_integrated/tools/run_integrated_matrix.sh diff --git a/TEAM_MODULE_INVENTORY.md b/TEAM_MODULE_INVENTORY.md new file mode 100644 index 000000000..bf769db56 --- /dev/null +++ b/TEAM_MODULE_INVENTORY.md @@ -0,0 +1,8 @@ +# Team Module Inventory + +- `RLA/`: Rank-low-rank adapter experiments and audited final evidence. Integrated flag: `model.rla_enabled`. +- `ESAM/`: Efficient/adaptive SAM training-loop variant. Integrated flag: `model.esam_enabled`. +- `MFB/`: member/feature bagging notebook implementation. Integrated flag: `model.mfb_enabled`. +- `cf_fisd_recovered/`: CF-FISD feature-importance diagnostics and model hooks. Integrated flag: `model.cf_fisd_enabled`. + +The integrated runner keeps the official TabM PLR baseline when all four flags are off and exposes module-only plus combined variants through `tabm_integrated/tools/generate_integrated_configs.py`. diff --git a/tabm_integrated/FINAL_EXPERIMENT_REPORT.md b/tabm_integrated/FINAL_EXPERIMENT_REPORT.md new file mode 100644 index 000000000..4a12a940b --- /dev/null +++ b/tabm_integrated/FINAL_EXPERIMENT_REPORT.md @@ -0,0 +1,56 @@ +# Final Integrated TabM Experiment Report + +Mean ± std over available seeds. Safe claims require all 3 seeds. + +| dataset | variant | metric | direction | inference | precision | mean ± std | delta | percent delta | n | status | config path | result path | +|---|---|---|---|---|---|---:|---:|---:|---:|---|---|---| +| sberbank-housing | baseline_plr | rmse | lower | mean | bfloat16 | 11.5057 ± 0.0432 | 0 | 0.000% | 3 | baseline | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation | +| sberbank-housing | rla_only | rmse | lower | mean | bfloat16 | 11.5041 ± 0.00471 | 0.00157171 | 0.014% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation | +| sberbank-housing | esam_only | rmse | lower | mean | bfloat16 | 11.5065 ± 0.0432 | -0.000818784 | -0.007% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation | +| sberbank-housing | mfb_only | rmse | lower | mean | bfloat16 | 11.5257 ± 0.0111 | -0.0199767 | -0.174% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation | +| sberbank-housing | cf_fisd_only | rmse | lower | mean | bfloat16 | 11.5355 ± 0.0054 | -0.0297966 | -0.259% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation | +| sberbank-housing | all_four_combined | rmse | lower | mean | bfloat16 | 11.5358 ± 0.00515 | -0.030061 | -0.261% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation | +| sberbank-housing | all_minus_rla | rmse | lower | mean | bfloat16 | 11.5352 ± 0.00412 | -0.0294744 | -0.256% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation | +| sberbank-housing | all_minus_esam | rmse | lower | mean | bfloat16 | 11.5344 ± 0.00432 | -0.028726 | -0.250% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation | +| sberbank-housing | all_minus_mfb | rmse | lower | mean | bfloat16 | 11.5315 ± 0.00449 | -0.0257712 | -0.224% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation | +| sberbank-housing | all_minus_cf_fisd | rmse | lower | mean | bfloat16 | 11.5117 ± 0.0154 | -0.00600246 | -0.052% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation | +| ecom-offers | baseline_plr | roc-auc | higher | mean | bfloat16 | 0.576461 ± 0.000301 | 0 | 0.000% | 3 | baseline | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation | +| ecom-offers | rla_only | roc-auc | higher | mean | bfloat16 | 0.576807 ± 0.000199 | 0.000345614 | 0.060% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation | +| ecom-offers | esam_only | roc-auc | higher | mean | bfloat16 | 0.5766 ± 0.00017 | 0.000138612 | 0.024% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation | +| ecom-offers | mfb_only | roc-auc | higher | mean | bfloat16 | 0.577706 ± 0.0034 | 0.00124474 | 0.216% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation | +| ecom-offers | cf_fisd_only | roc-auc | higher | mean | bfloat16 | 0.573314 ± 0.000392 | -0.00314765 | -0.546% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation | +| ecom-offers | all_four_combined | roc-auc | higher | mean | bfloat16 | 0.575357 ± 0.00178 | -0.00110457 | -0.192% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation | +| ecom-offers | all_minus_rla | roc-auc | higher | mean | bfloat16 | 0.572037 ± 0.00203 | -0.0044245 | -0.768% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation | +| ecom-offers | all_minus_esam | roc-auc | higher | mean | bfloat16 | 0.572005 ± 0.00109 | -0.00445646 | -0.773% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation | +| ecom-offers | all_minus_mfb | roc-auc | higher | mean | bfloat16 | 0.574369 ± 0.00047 | -0.00209235 | -0.363% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation | +| ecom-offers | all_minus_cf_fisd | roc-auc | higher | mean | bfloat16 | 0.576756 ± 0.00348 | 0.000294538 | 0.051% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation | +| homesite-insurance | baseline_plr | roc-auc | higher | mean | bfloat16 | 0.953231 ± 0.0006 | 0 | 0.000% | 3 | baseline | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation | +| homesite-insurance | rla_only | roc-auc | higher | mean | bfloat16 | 0.951962 ± 0.000732 | -0.00126848 | -0.133% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation | +| homesite-insurance | esam_only | roc-auc | higher | mean | bfloat16 | 0.953275 ± 0.00054 | 4.45755e-05 | 0.005% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation | +| homesite-insurance | mfb_only | roc-auc | higher | mean | bfloat16 | 0.950035 ± 0.000335 | -0.00319556 | -0.335% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation | +| homesite-insurance | cf_fisd_only | roc-auc | higher | mean | bfloat16 | 0.954365 ± 0.000453 | 0.001134 | 0.119% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation | +| homesite-insurance | all_four_combined | roc-auc | higher | mean | bfloat16 | 0.949588 ± 0.000717 | -0.00364296 | -0.382% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation | +| homesite-insurance | all_minus_rla | roc-auc | higher | mean | bfloat16 | 0.95047 ± 0.000475 | -0.00276044 | -0.290% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation | +| homesite-insurance | all_minus_esam | roc-auc | higher | mean | bfloat16 | 0.949443 ± 0.00153 | -0.0037874 | -0.397% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation | +| homesite-insurance | all_minus_mfb | roc-auc | higher | mean | bfloat16 | 0.95332 ± 0.000513 | 8.89334e-05 | 0.009% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation | +| homesite-insurance | all_minus_cf_fisd | roc-auc | higher | mean | bfloat16 | 0.94591 ± 0.00176 | -0.00732119 | -0.768% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation | +| cooking-time | baseline_plr | rmse | lower | mean | bfloat16 | 2.56457 ± 0.0029 | 0 | 0.000% | 3 | baseline | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation | +| cooking-time | rla_only | rmse | lower | mean | bfloat16 | 2.56088 ± 0.0138 | 0.00368773 | 0.144% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation | +| cooking-time | esam_only | rmse | lower | mean | bfloat16 | 2.56386 ± 0.00279 | 0.000706405 | 0.028% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation | +| cooking-time | mfb_only | rmse | lower | mean | bfloat16 | 2.5445 ± 0.00955 | 0.0200701 | 0.783% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation | +| cooking-time | cf_fisd_only | rmse | lower | mean | bfloat16 | 2.56324 ± 0.00279 | 0.00133126 | 0.052% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation | +| cooking-time | all_four_combined | rmse | lower | mean | bfloat16 | 2.54421 ± 0.00684 | 0.0203545 | 0.794% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation | +| cooking-time | all_minus_rla | rmse | lower | mean | bfloat16 | 2.54693 ± 0.00325 | 0.0176392 | 0.688% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation | +| cooking-time | all_minus_esam | rmse | lower | mean | bfloat16 | 2.54322 ± 0.00473 | 0.0213521 | 0.833% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation | +| cooking-time | all_minus_mfb | rmse | lower | mean | bfloat16 | 2.55993 ± 0.0125 | 0.00464007 | 0.181% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation | +| cooking-time | all_minus_cf_fisd | rmse | lower | mean | bfloat16 | 2.54518 ± 0.0029 | 0.0193905 | 0.756% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation | +| delivery-eta | baseline_plr | rmse | lower | mean | bfloat16 | 1.85423 ± 0.0129 | 0 | 0.000% | 3 | baseline | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation | +| delivery-eta | rla_only | rmse | lower | mean | bfloat16 | 1.87913 ± 0.00475 | -0.0249039 | -1.343% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation | +| delivery-eta | esam_only | rmse | lower | mean | bfloat16 | 1.85645 ± 0.0134 | -0.00222041 | -0.120% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation | +| delivery-eta | mfb_only | rmse | lower | mean | bfloat16 | 1.88817 ± 0.0102 | -0.0339417 | -1.831% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation | +| delivery-eta | cf_fisd_only | rmse | lower | mean | bfloat16 | 1.85934 ± 0.0133 | -0.00511517 | -0.276% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation | +| delivery-eta | all_four_combined | rmse | lower | mean | bfloat16 | 1.8809 ± 0.0154 | -0.0266763 | -1.439% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation | +| delivery-eta | all_minus_rla | rmse | lower | mean | bfloat16 | 1.87905 ± 0.0243 | -0.0248174 | -1.338% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation | +| delivery-eta | all_minus_esam | rmse | lower | mean | bfloat16 | 1.87891 ± 0.0186 | -0.0246832 | -1.331% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation | +| delivery-eta | all_minus_mfb | rmse | lower | mean | bfloat16 | 1.86875 ± 0.00678 | -0.0145174 | -0.783% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation | +| delivery-eta | all_minus_cf_fisd | rmse | lower | mean | bfloat16 | 1.87292 ± 0.0162 | -0.0186869 | -1.008% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation | diff --git a/tabm_integrated/__init__.py b/tabm_integrated/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/bin/run_integrated.py b/tabm_integrated/paper/bin/run_integrated.py new file mode 100644 index 000000000..448d6961d --- /dev/null +++ b/tabm_integrated/paper/bin/run_integrated.py @@ -0,0 +1,198 @@ +from __future__ import annotations +import argparse, json, math, random, time, tomllib +from pathlib import Path +from typing import Any +import numpy as np +import torch +from sklearn.metrics import mean_squared_error, roc_auc_score +from torch import nn +from torch.utils.data import DataLoader, TensorDataset +import tabm +import rtdl_num_embeddings + +LOWER = {'rmse'} + +def load_toml(path: Path) -> dict[str, Any]: + with path.open('rb') as f: + return tomllib.load(f) + +def dump_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + '\n') + +def set_seed(seed: int) -> None: + random.seed(seed); np.random.seed(seed); torch.manual_seed(seed); torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.benchmark = True + +def read_arrays(data_path: Path): + info = json.loads((data_path / 'info.json').read_text()) + x_num, x_cat, y = {}, {}, {} + for part in ['train', 'val', 'test']: + nums = [] + for name in ['X_num', 'X_bin']: + p = data_path / f'{name}_{part}.npy' + if p.exists(): + nums.append(np.load(p, allow_pickle=True).astype('float32')) + y[part] = np.load(data_path / f'Y_{part}.npy', allow_pickle=True) + x_num[part] = np.concatenate(nums, axis=1).astype('float32') if nums else np.zeros((len(y[part]), 0), dtype='float32') + pcat = data_path / f'X_cat_{part}.npy' + if pcat.exists(): + x_cat[part] = np.load(pcat, allow_pickle=True).astype('int64') + return x_num, (x_cat if x_cat else None), y, info + +def standardize_num(x): + if x['train'].shape[1] == 0: + return x + mean = np.nanmean(x['train'], axis=0, keepdims=True); mean = np.nan_to_num(mean, nan=0.0) + std = np.nanstd(x['train'], axis=0, keepdims=True); std = np.nan_to_num(std, nan=0.0) + span = np.nanmax(x['train'], axis=0) - np.nanmin(x['train'], axis=0); span = np.nan_to_num(span, nan=0.0) + keep = (std.reshape(-1) >= 1e-6) & (span >= 1e-6) + if not keep.any(): + return {k: np.zeros((v.shape[0], 0), dtype='float32') for k, v in x.items()} + mean = mean[:, keep]; std = std[:, keep] + return {k: np.nan_to_num((v[:, keep] - mean) / std, nan=0.0, posinf=0.0, neginf=0.0).astype('float32') for k, v in x.items()} + +def cat_cardinalities(x_cat): + if not x_cat: + return None + return [max(int(x_cat[p][:, j].max()) for p in x_cat) + 1 for j in range(x_cat['train'].shape[1])] + +def feature_importance(x_num, y): + if x_num.shape[1] == 0: + return np.zeros((0,), dtype='float32') + yy = y.astype('float32'); yy = (yy - yy.mean()) / (yy.std() + 1e-6) + scores = np.nan_to_num(np.abs((x_num.astype('float32') * yy[:, None]).mean(0)), nan=0.0) + if float(scores.max(initial=0.0)) <= 0: + scores = np.ones_like(scores) + return np.clip(scores / (scores.mean() + 1e-6), 0.25, 4.0).astype('float32') + +class IntegratedTabM(nn.Module): + def __init__(self, cfg, n_num_features, cards, d_out, bins, cf_weights): + super().__init__() + mc = cfg['model'] + num_embeddings = None + if n_num_features and bins is not None: + num_embeddings = rtdl_num_embeddings.PiecewiseLinearEmbeddings(bins, int(mc.get('d_embedding', 16)), activation=True, version='B') + self.model = tabm.TabM( + n_num_features=n_num_features, + cat_cardinalities=cards, + d_out=d_out, + num_embeddings=num_embeddings, + arch_type=mc.get('arch_type', 'tabm'), + k=int(mc.get('k', 16)), + n_blocks=int(mc.get('n_blocks', 2)), + d_block=int(mc.get('d_block', 384)), + dropout=float(mc.get('dropout', 0.1)), + start_scaling_init=mc.get('start_scaling_init', 'random-signs'), + ) + self.rla_enabled = bool(mc.get('rla_enabled', False)) and n_num_features > 0 + self.rla_scale = float(mc.get('rla_scale', 0.05)) + if self.rla_enabled: + rank = int(mc.get('rla_rank', 4)) + self.rla_a = nn.Parameter(torch.randn(n_num_features, rank) * 1e-3) + self.rla_b = nn.Parameter(torch.zeros(rank, n_num_features)) + self.mfb_enabled = bool(mc.get('mfb_enabled', False)) and n_num_features > 0 + self.mfb_keep = float(mc.get('mfb_keep', 0.8)) + self.cf_enabled = bool(mc.get('cf_fisd_enabled', False)) and cf_weights is not None and n_num_features > 0 + self.register_buffer('cf_weights', cf_weights if cf_weights is not None else torch.ones(n_num_features)) + def forward(self, x_num, x_cat): + if x_num is not None and x_num.shape[1] == 0: + x_num = None + if x_num is not None and self.cf_enabled: + x_num = x_num * self.cf_weights[None, :] + if x_num is not None and self.rla_enabled: + x_num = x_num + self.rla_scale * (x_num @ self.rla_a @ self.rla_b) + if self.training and x_num is not None and self.mfb_enabled: + mask = (torch.rand((1, x_num.shape[1]), device=x_num.device) < self.mfb_keep).to(x_num.dtype) / max(self.mfb_keep, 1e-6) + x_num = x_num * mask + return self.model(x_num, x_cat) + +def make_loaders(x_num, x_cat, y, task_type, batch_size): + out = {} + for part in ['train', 'val', 'test']: + xc = torch.as_tensor(x_cat[part], dtype=torch.long) if x_cat is not None else torch.empty((len(y[part]), 0), dtype=torch.long) + ds = TensorDataset(torch.as_tensor(x_num[part], dtype=torch.float32), xc, torch.as_tensor(y[part], dtype=torch.float32)) + out[part] = DataLoader(ds, batch_size=batch_size if part == 'train' else batch_size * 4, shuffle=(part == 'train'), num_workers=2, pin_memory=True) + return out + +def predict(model, loader, device, task_type, y_mean, y_std): + model.eval(); preds=[]; targets=[] + with torch.no_grad(): + for xb, xc, yb in loader: + xb=xb.to(device, non_blocking=True); xc=xc.to(device, non_blocking=True) + out = model(xb, xc if xc.shape[1] else None).mean(1).squeeze(-1) + out = torch.sigmoid(out) if task_type == 'binclass' else out * y_std + y_mean + preds.append(out.float().cpu().numpy()); targets.append(yb.numpy()) + return np.concatenate(preds), np.concatenate(targets) + +def metric(task_type, pred, target): + if task_type == 'binclass': + auc = float(roc_auc_score(target.astype(int), pred)); return {'roc-auc': auc, 'score': auc} + rmse = float(math.sqrt(mean_squared_error(target.astype('float32'), pred.astype('float32')))); return {'rmse': rmse, 'score': -rmse} + +def train_one(config_path: Path, output: Path, force=False): + if output.exists() and not force and (output/'DONE').exists() and (output/'report.json').exists(): + return + output.mkdir(parents=True, exist_ok=True) + cfg=load_toml(config_path); set_seed(int(cfg.get('seed',0))) + device=torch.device('cuda' if torch.cuda.is_available() else 'cpu') + x_num,x_cat,y_raw,info=read_arrays((Path.cwd()/cfg['data']['path']).resolve()) + x_num=standardize_num(x_num); task_type=info['task_type']; y={k:v.copy() for k,v in y_raw.items()} + y_mean=0.0; y_std=1.0 + if task_type == 'regression': + y_mean=float(y['train'].mean()); y_std=float(y['train'].std()+1e-6) + y={k:((v.astype('float32')-y_mean)/y_std).astype('float32') for k,v in y.items()} + cards=cat_cardinalities(x_cat); n_num=x_num['train'].shape[1] + bins = rtdl_num_embeddings.compute_bins(torch.as_tensor(x_num['train'], dtype=torch.float32), n_bins=int(cfg['model'].get('n_bins',48))) if n_num else None + cf = torch.as_tensor(feature_importance(x_num['train'], y_raw['train']), dtype=torch.float32) if cfg['model'].get('cf_fisd_enabled', False) else None + model=IntegratedTabM(cfg,n_num,cards,1,bins,cf).to(device) + loaders=make_loaders(x_num,x_cat,y,task_type,int(cfg.get('batch_size',1024))) + opt=torch.optim.AdamW(model.parameters(), lr=float(cfg['optimizer'].get('lr',3e-4)), weight_decay=float(cfg['optimizer'].get('weight_decay',1e-5))) + loss_fn=nn.BCEWithLogitsLoss() if task_type=='binclass' else nn.MSELoss(); amp=bool(cfg.get('amp',True)) and device.type=='cuda' + rho=float(cfg['model'].get('esam_rho',0.0)) if cfg['model'].get('esam_enabled',False) else 0.0 + best=None; best_score=-1e100; bad=0; history=[]; start=time.time() + for epoch in range(int(cfg.get('n_epochs',20))): + model.train(); losses=[] + for xb,xc,yb in loaders['train']: + xb=xb.to(device,non_blocking=True); xc=xc.to(device,non_blocking=True); yb=yb.to(device,non_blocking=True) + opt.zero_grad(set_to_none=True) + with torch.autocast(device_type='cuda', dtype=torch.bfloat16, enabled=amp): + logits=model(xb, xc if xc.shape[1] else None).squeeze(-1); target=yb[:,None].expand_as(logits); loss=loss_fn(logits,target) + if not torch.isfinite(loss): raise RuntimeError(f'NaN/Inf loss at epoch={epoch}') + loss.backward() + if rho > 0: + grads=[p.grad for p in model.parameters() if p.grad is not None] + norm=torch.norm(torch.stack([g.detach().norm() for g in grads])) if grads else torch.tensor(0.0,device=device); scale=rho/(norm+1e-12); eps=[] + with torch.no_grad(): + for p in model.parameters(): + if p.grad is None: eps.append(None) + else: + e=p.grad*scale; p.add_(e); eps.append(e) + opt.zero_grad(set_to_none=True) + with torch.autocast(device_type='cuda', dtype=torch.bfloat16, enabled=amp): + logits2=model(xb, xc if xc.shape[1] else None).squeeze(-1); loss2=loss_fn(logits2,target) + if not torch.isfinite(loss2): raise RuntimeError(f'NaN/Inf ESAM loss at epoch={epoch}') + loss2.backward() + with torch.no_grad(): + for p,e in zip(model.parameters(),eps): + if e is not None: p.sub_(e) + losses.append(float(loss2.detach().cpu())) + else: + losses.append(float(loss.detach().cpu())) + torch.nn.utils.clip_grad_norm_(model.parameters(), float(cfg.get('gradient_clipping_norm',1.0))); opt.step() + vp,vy=predict(model,loaders['val'],device,task_type,y_mean,y_std); tp,ty=predict(model,loaders['test'],device,task_type,y_mean,y_std) + metrics={'val':metric(task_type,vp,vy),'test':metric(task_type,tp,ty)}; score=metrics['val']['score'] + history.append({'epoch':epoch,'train_loss':float(np.mean(losses)),'val_score':score,'test_score':metrics['test']['score']}) + if score > best_score: best_score=score; best={'metrics':metrics,'best_epoch':epoch}; bad=0 + else: + bad += 1 + if bad >= int(cfg.get('patience',6)): break + report={'dataset':cfg['dataset'],'variant':cfg['variant'],'seed':cfg['seed'],'config_path':str(config_path),'result_path':str(output),'gpu_name':torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'cpu','amp_enabled':amp,'amp_dtype':'bfloat16' if amp else 'fp32','inference_mode':'mean','task_type':task_type,'metric_direction':'lower' if task_type=='regression' else 'higher','config':cfg,'time_seconds':time.time()-start,'history':history,**best} + dump_json(output/'report.json',report); (output/'DONE').write_text('done\n') + +def main(): + p=argparse.ArgumentParser(); p.add_argument('config',type=Path); p.add_argument('output',type=Path); p.add_argument('--force',action='store_true'); a=p.parse_args() + try: train_one(a.config,a.output,a.force) + except Exception as e: + a.output.mkdir(parents=True, exist_ok=True); dump_json(a.output/'report.json', {'failure':repr(e),'config_path':str(a.config),'result_path':str(a.output)}); raise +if __name__ == '__main__': main() diff --git a/tabm_integrated/paper/data/.gitignore b/tabm_integrated/paper/data/.gitignore new file mode 100644 index 000000000..d6b7ef32c --- /dev/null +++ b/tabm_integrated/paper/data/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/tabm_integrated/paper/exp/.gitignore b/tabm_integrated/paper/exp/.gitignore new file mode 100644 index 000000000..9d33728e2 --- /dev/null +++ b/tabm_integrated/paper/exp/.gitignore @@ -0,0 +1,11 @@ +_logs/ +_queue_*.txt +_queue_*.txt.lock +_queue_*.txt.tmp +*.out +FAILED +*.pt +*.pth +checkpoint* +cache/ +__pycache__/ diff --git a/tabm_integrated/paper/exp/final_integrated_audit.csv b/tabm_integrated/paper/exp/final_integrated_audit.csv new file mode 100644 index 000000000..2685aa3b9 --- /dev/null +++ b/tabm_integrated/paper/exp/final_integrated_audit.csv @@ -0,0 +1,151 @@ +dataset,variant,seed,result_path,report_json_present,DONE_present,failure_block_absent,gpu_name,amp_dtype,inference_mode,metric,test_metric,failure +sberbank-housing,baseline_plr,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.546495072614636, +sberbank-housing,baseline_plr,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.510137982108967, +sberbank-housing,baseline_plr,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.460434895463788, +sberbank-housing,rla_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.49868567875769, +sberbank-housing,rla_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.506723180155442, +sberbank-housing,rla_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.506943969685626, +sberbank-housing,esam_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.547765632938749, +sberbank-housing,esam_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.510244699059466, +sberbank-housing,esam_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.461513970249932, +sberbank-housing,mfb_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.527389417691648, +sberbank-housing,mfb_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.535810359671178, +sberbank-housing,mfb_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.513798272472021, +sberbank-housing,cf_fisd_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.541630897512452, +sberbank-housing,cf_fisd_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.531481567373374, +sberbank-housing,cf_fisd_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.533345184614086, +sberbank-housing,all_four_combined,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.529940564415122, +sberbank-housing,all_four_combined,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.5397613458401, +sberbank-housing,all_four_combined,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.537548960274977, +sberbank-housing,all_minus_rla,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.530414333922007, +sberbank-housing,all_minus_rla,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.537306272919873, +sberbank-housing,all_minus_rla,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.537770482423289, +sberbank-housing,all_minus_esam,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.529550814849099, +sberbank-housing,all_minus_esam,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.535907580086148, +sberbank-housing,all_minus_esam,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.53778767500918, +sberbank-housing,all_minus_mfb,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.529854542723736, +sberbank-housing,all_minus_mfb,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.527989037555509, +sberbank-housing,all_minus_mfb,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.53653783950218, +sberbank-housing,all_minus_cf_fisd,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.495833595850131, +sberbank-housing,all_minus_cf_fisd,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.52660642338622, +sberbank-housing,all_minus_cf_fisd,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.512635297114862, +ecom-offers,baseline_plr,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5767959687132622, +ecom-offers,baseline_plr,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5763749354641947, +ecom-offers,baseline_plr,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.57621293188747, +ecom-offers,rla_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5766629173713693, +ecom-offers,rla_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5767242343675878, +ecom-offers,rla_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5770335251939862, +ecom-offers,esam_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5766903074687046, +ecom-offers,esam_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5767053647384629, +ecom-offers,esam_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5764039992802762, +ecom-offers,mfb_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5756126061669247, +ecom-offers,mfb_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5758805406428937, +ecom-offers,mfb_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5816249076525681, +ecom-offers,cf_fisd_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5731309058948163, +ecom-offers,cf_fisd_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5730458902809421, +ecom-offers,cf_fisd_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5737641046913308, +ecom-offers,all_four_combined,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.575590289916974, +ecom-offers,all_four_combined,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5770100559377882, +ecom-offers,all_four_combined,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5734697781503159, +ecom-offers,all_minus_rla,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5697199786711201, +ecom-offers,all_minus_rla,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.573474359180514, +ecom-offers,all_minus_rla,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5729159973083131, +ecom-offers,all_minus_esam,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.573248515631535, +ecom-offers,all_minus_esam,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5715625167488105, +ecom-offers,all_minus_esam,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5712034110933558, +ecom-offers,all_minus_mfb,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5743883676894275, +ecom-offers,all_minus_mfb,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5738892213665745, +ecom-offers,all_minus_mfb,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5748292097098053, +ecom-offers,all_minus_cf_fisd,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.574504443183962, +ecom-offers,all_minus_cf_fisd,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5749947559912094, +ecom-offers,all_minus_cf_fisd,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5807682519060278, +homesite-insurance,baseline_plr,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9527329104991943, +homesite-insurance,baseline_plr,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9530616509300706, +homesite-insurance,baseline_plr,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9538975705174861, +homesite-insurance,rla_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9511202086000653, +homesite-insurance,rla_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9523188529200016, +homesite-insurance,rla_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9524476380801628, +homesite-insurance,esam_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9528578176071593, +homesite-insurance,esam_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9530825030113498, +homesite-insurance,esam_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9538855376826346, +homesite-insurance,mfb_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9503995432741815, +homesite-insurance,mfb_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.94973945288198, +homesite-insurance,mfb_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9499664578687927, +homesite-insurance,cf_fisd_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9543305592735997, +homesite-insurance,cf_fisd_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9539302138104276, +homesite-insurance,cf_fisd_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9548333529692444, +homesite-insurance,all_four_combined,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9488481290971439, +homesite-insurance,all_four_combined,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9496361106437045, +homesite-insurance,all_four_combined,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9502789974647928, +homesite-insurance,all_minus_rla,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.950880361340715, +homesite-insurance,all_minus_rla,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9499504624136281, +homesite-insurance,all_minus_rla,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9505799995534802, +homesite-insurance,all_minus_esam,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9485440704751056, +homesite-insurance,all_minus_esam,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9512055136672225, +homesite-insurance,all_minus_esam,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9485803381158939, +homesite-insurance,all_minus_mfb,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9532358612499129, +homesite-insurance,all_minus_mfb,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9528538308245276, +homesite-insurance,all_minus_mfb,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.953869240198483, +homesite-insurance,all_minus_cf_fisd,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9438768769410195, +homesite-insurance,all_minus_cf_fisd,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9468154377695126, +homesite-insurance,all_minus_cf_fisd,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.947036245121507, +cooking-time,baseline_plr,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5612615058011223, +cooking-time,baseline_plr,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5657847681143955, +cooking-time,baseline_plr,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5666563240272247, +cooking-time,rla_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5699112877582024, +cooking-time,rla_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5449543831564863, +cooking-time,rla_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5677737420490234, +cooking-time,esam_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5611478448009337, +cooking-time,esam_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.563722179453466, +cooking-time,esam_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.56671335830164, +cooking-time,mfb_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5555160998555837, +cooking-time,mfb_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.539276959092014, +cooking-time,mfb_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5386993618023146, +cooking-time,cf_fisd_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.563118185514718, +cooking-time,cf_fisd_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5605115851699063, +cooking-time,cf_fisd_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.566079036110905, +cooking-time,all_four_combined,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5511833212856643, +cooking-time,all_four_combined,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.537508726809674, +cooking-time,all_four_combined,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.543946905739664, +cooking-time,all_minus_rla,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.550683292768776, +cooking-time,all_minus_rla,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5451569174185407, +cooking-time,all_minus_rla,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.544944827487872, +cooking-time,all_minus_esam,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5437589902976687, +cooking-time,all_minus_esam,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.538235385864352, +cooking-time,all_minus_esam,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5476520499692263, +cooking-time,all_minus_mfb,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.572318549397406, +cooking-time,all_minus_mfb,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5472973432410306, +cooking-time,all_minus_mfb,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5601664827274426, +cooking-time,all_minus_cf_fisd,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.546692730518933, +cooking-time,all_minus_cf_fisd,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5418282337637406, +cooking-time,all_minus_cf_fisd,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.547009985219361, +delivery-eta,baseline_plr,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8396023796480883, +delivery-eta,baseline_plr,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8639852605326992, +delivery-eta,baseline_plr,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8590972436481992, +delivery-eta,rla_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8738282038863792, +delivery-eta,rla_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8829804025076913, +delivery-eta,rla_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.88058796129553, +delivery-eta,esam_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8414060273006467, +delivery-eta,esam_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.867077237628412, +delivery-eta,esam_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8608628418981936, +delivery-eta,mfb_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.882313199140896, +delivery-eta,mfb_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8999776362056646, +delivery-eta,mfb_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8822192131865638, +delivery-eta,cf_fisd_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.847462746808535, +delivery-eta,cf_fisd_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8568220804711988, +delivery-eta,cf_fisd_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8737455622217765, +delivery-eta,all_four_combined,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8656861082768745, +delivery-eta,all_four_combined,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8804933819744376, +delivery-eta,all_four_combined,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8965344093393364, +delivery-eta,all_minus_rla,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.860688971411864, +delivery-eta,all_minus_rla,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8698356715330888, +delivery-eta,all_minus_rla,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.9066122992751606, +delivery-eta,all_minus_esam,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8574282929864145, +delivery-eta,all_minus_esam,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8896128569809156, +delivery-eta,all_minus_esam,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.889693353794343, +delivery-eta,all_minus_mfb,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8611355949170485, +delivery-eta,all_minus_mfb,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8741570167185646, +delivery-eta,all_minus_mfb,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.870944405411248, +delivery-eta,all_minus_cf_fisd,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.891667480580607, +delivery-eta,all_minus_cf_fisd,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8636960381640717, +delivery-eta,all_minus_cf_fisd,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8633819488889196, diff --git a/tabm_integrated/paper/exp/final_integrated_summary.csv b/tabm_integrated/paper/exp/final_integrated_summary.csv new file mode 100644 index 000000000..582389d43 --- /dev/null +++ b/tabm_integrated/paper/exp/final_integrated_summary.csv @@ -0,0 +1,51 @@ +dataset,variant,metric,metric_direction,inference_mode,precision,n_seeds,mean,std,baseline_mean,absolute_delta,percent_delta,safe_claim_status,config_path,result_path +sberbank-housing,baseline_plr,rmse,lower,mean,bfloat16,3,11.50568931672913,0.043202216269019954,11.50568931672913,0.0,0.0,baseline,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation +sberbank-housing,rla_only,rmse,lower,mean,bfloat16,3,11.50411760953292,0.00470548519967073,11.50568931672913,0.0015717071962111362,0.013660261049513072,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation +sberbank-housing,esam_only,rmse,lower,mean,bfloat16,3,11.506508100749382,0.04324706873702009,11.50568931672913,-0.0008187840202520391,-0.007116340427005423,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation +sberbank-housing,mfb_only,rmse,lower,mean,bfloat16,3,11.525666016611616,0.011106780774725373,11.50568931672913,-0.01997669988248596,-0.17362453767494038,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation +sberbank-housing,cf_fisd_only,rmse,lower,mean,bfloat16,3,11.535485883166638,0.005402700092330267,11.50568931672913,-0.02979656643750772,-0.258972458035903,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation +sberbank-housing,all_four_combined,rmse,lower,mean,bfloat16,3,11.535750290176733,0.005151538366452624,11.50568931672913,-0.03006097344760228,-0.26127051252717204,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation +sberbank-housing,all_minus_rla,rmse,lower,mean,bfloat16,3,11.535163696421723,0.004119612327761725,11.50568931672913,-0.02947437969259248,-0.25617221951002184,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation +sberbank-housing,all_minus_esam,rmse,lower,mean,bfloat16,3,11.534415356648143,0.004316423799383681,11.50568931672913,-0.02872603991901279,-0.24966813485260272,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation +sberbank-housing,all_minus_mfb,rmse,lower,mean,bfloat16,3,11.531460473260475,0.004494970895064652,11.50568931672913,-0.025771156531344985,-0.2239862021467418,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation +sberbank-housing,all_minus_cf_fisd,rmse,lower,mean,bfloat16,3,11.511691772117072,0.015408095541226104,11.50568931672913,-0.006002455387941197,-0.0521694548036657,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation +ecom-offers,baseline_plr,roc-auc,higher,mean,bfloat16,3,0.576461278688309,0.0003009557265118819,0.576461278688309,0.0,0.0,baseline,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation +ecom-offers,rla_only,roc-auc,higher,mean,bfloat16,3,0.5768068923109811,0.00019864992134586008,0.576461278688309,0.00034561362267215845,0.05995435174042813,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation +ecom-offers,esam_only,roc-auc,higher,mean,bfloat16,3,0.5765998904958146,0.0001698137408662012,0.576461278688309,0.00013861180750562863,0.024045293696226846,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation +ecom-offers,mfb_only,roc-auc,higher,mean,bfloat16,3,0.5777060181541288,0.003396500904841115,0.576461278688309,0.0012447394658198219,0.215927680112726,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation +ecom-offers,cf_fisd_only,roc-auc,higher,mean,bfloat16,3,0.5733136336223631,0.00039242840319386274,0.576461278688309,-0.003147645065945892,-0.5460288803973277,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation +ecom-offers,all_four_combined,roc-auc,higher,mean,bfloat16,3,0.5753567080016927,0.0017816599244741948,0.576461278688309,-0.0011045706866162908,-0.19161229512754993,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation +ecom-offers,all_minus_rla,roc-auc,higher,mean,bfloat16,3,0.5720367783866491,0.002025737566045578,0.576461278688309,-0.004424500301659862,-0.7675277534212626,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation +ecom-offers,all_minus_esam,roc-auc,higher,mean,bfloat16,3,0.5720048144912337,0.0010919402972391877,0.576461278688309,-0.004456464197075238,-0.7730726003341564,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation +ecom-offers,all_minus_mfb,roc-auc,higher,mean,bfloat16,3,0.5743689329219358,0.00047029544330458393,0.576461278688309,-0.0020923457663731826,-0.3629638006448839,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation +ecom-offers,all_minus_cf_fisd,roc-auc,higher,mean,bfloat16,3,0.5767558170270664,0.0034835078449268714,0.576461278688309,0.00029453833875736635,0.051094210425991576,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation +homesite-insurance,baseline_plr,roc-auc,higher,mean,bfloat16,3,0.953230710648917,0.0006004532712494962,0.953230710648917,0.0,0.0,baseline,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation +homesite-insurance,rla_only,roc-auc,higher,mean,bfloat16,3,0.9519622332000766,0.0007320522349105445,0.953230710648917,-0.0012684774488403994,-0.13307139968002882,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation +homesite-insurance,esam_only,roc-auc,higher,mean,bfloat16,3,0.9532752861003813,0.0005403018859361379,0.953230710648917,4.457545146430153e-05,0.004676250037512593,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation +homesite-insurance,mfb_only,roc-auc,higher,mean,bfloat16,3,0.9500351513416514,0.00033536387160697765,0.953230710648917,-0.003195559307265561,-0.3352346154573814,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation +homesite-insurance,cf_fisd_only,roc-auc,higher,mean,bfloat16,3,0.9543647086844239,0.0004525369838386773,0.953230710648917,0.0011339980355069557,0.11896364886680795,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation +homesite-insurance,all_four_combined,roc-auc,higher,mean,bfloat16,3,0.9495877457352138,0.0007166592249211433,0.953230710648917,-0.003642964913703195,-0.3821703259248988,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation +homesite-insurance,all_minus_rla,roc-auc,higher,mean,bfloat16,3,0.9504702744359411,0.0004745605385127754,0.953230710648917,-0.0027604362129758364,-0.28958741909360586,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation +homesite-insurance,all_minus_esam,roc-auc,higher,mean,bfloat16,3,0.9494433074194073,0.0015262231096144455,0.953230710648917,-0.003787403229509656,-0.3973228293212837,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation +homesite-insurance,all_minus_mfb,roc-auc,higher,mean,bfloat16,3,0.9533196440909745,0.0005128632590870418,0.953230710648917,8.893344205751497e-05,0.009329687038405744,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation +homesite-insurance,all_minus_cf_fisd,roc-auc,higher,mean,bfloat16,3,0.9459095199440131,0.0017637792250980726,0.953230710648917,-0.0073211907049038905,-0.7680397434866477,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation +cooking-time,baseline_plr,rmse,lower,mean,bfloat16,3,2.564567532647581,0.0028960770983219863,2.564567532647581,0.0,0.0,baseline,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation +cooking-time,rla_only,rmse,lower,mean,bfloat16,3,2.5608798043212375,0.013833168648965952,2.564567532647581,0.0036877283263434535,0.1437953292084438,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation +cooking-time,esam_only,rmse,lower,mean,bfloat16,3,2.56386112751868,0.0027853572581009772,2.564567532647581,0.0007064051289011886,0.02754480511464315,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation +cooking-time,mfb_only,rmse,lower,mean,bfloat16,3,2.544497473583304,0.009546779475477335,2.564567532647581,0.020070059064277057,0.7825903903399004,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation +cooking-time,cf_fisd_only,rmse,lower,mean,bfloat16,3,2.5632362689318433,0.002785603213876169,2.564567532647581,0.0013312637157376628,0.05190987169533831,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation +cooking-time,all_four_combined,rmse,lower,mean,bfloat16,3,2.5442129846116677,0.006841179137786861,2.564567532647581,0.0203545480359133,0.793683448643674,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation +cooking-time,all_minus_rla,rmse,lower,mean,bfloat16,3,2.54692834589173,0.0032536080083122717,2.564567532647581,0.01763918675585119,0.68780355874041,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation +cooking-time,all_minus_esam,rmse,lower,mean,bfloat16,3,2.543215475377082,0.0047318016724798495,2.564567532647581,0.021352057270498825,0.8325792555150855,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation +cooking-time,all_minus_mfb,rmse,lower,mean,bfloat16,3,2.559927458455293,0.012512315486430373,2.564567532647581,0.004640074192288068,0.18093008404804212,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation +cooking-time,all_minus_cf_fisd,rmse,lower,mean,bfloat16,3,2.545176983167345,0.002904437047405368,2.564567532647581,0.019390549480235997,0.7560943212993806,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation +delivery-eta,baseline_plr,rmse,lower,mean,bfloat16,3,1.8542282946096622,0.012900047232982671,1.8542282946096622,0.0,0.0,baseline,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation +delivery-eta,rla_only,rmse,lower,mean,bfloat16,3,1.879132189229867,0.004746592370465458,1.8542282946096622,-0.024903894620204703,-1.3430867543441989,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation +delivery-eta,esam_only,rmse,lower,mean,bfloat16,3,1.8564487022757508,0.013392767875906186,1.8542282946096622,-0.002220407666088553,-0.11974834342369778,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation +delivery-eta,mfb_only,rmse,lower,mean,bfloat16,3,1.8881700161777082,0.01022580688177479,1.8542282946096622,-0.03394172156804598,-1.8305039172747133,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation +delivery-eta,cf_fisd_only,rmse,lower,mean,bfloat16,3,1.8593434631671701,0.013321584910823822,1.8542282946096622,-0.005115168557507932,-0.27586509020372474,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation +delivery-eta,all_four_combined,rmse,lower,mean,bfloat16,3,1.8809046331968828,0.015428261901006317,1.8542282946096622,-0.026676338587220627,-1.438676060804925,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation +delivery-eta,all_minus_rla,rmse,lower,mean,bfloat16,3,1.8790456474067045,0.024307524600509958,1.8542282946096622,-0.024817352797042336,-1.3384194853021965,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation +delivery-eta,all_minus_esam,rmse,lower,mean,bfloat16,3,1.878911501253891,0.018605047649282478,1.8542282946096622,-0.02468320664422885,-1.3311848770717292,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation +delivery-eta,all_minus_mfb,rmse,lower,mean,bfloat16,3,1.8687456723489537,0.0067834487351692615,1.8542282946096622,-0.014517377739291515,-0.7829336755077185,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation +delivery-eta,all_minus_cf_fisd,rmse,lower,mean,bfloat16,3,1.872915155877866,0.01624074888470241,1.8542282946096622,-0.018686861268203847,-1.0077972233800725,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..eb4b802dd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0.toml @@ -0,0 +1,35 @@ +dataset = "cooking-time" +variant = "all_four_combined" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..1533a55d0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0/report.json @@ -0,0 +1,136 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 5, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_four_combined" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.597904784280686, + "train_loss": 0.5734070920729423, + "val_score": -2.6454041701159015 + }, + { + "epoch": 1, + "test_score": -2.5479064912091176, + "train_loss": 0.519261709741644, + "val_score": -2.5936452086088506 + }, + { + "epoch": 2, + "test_score": -2.568095726724589, + "train_loss": 0.5110039501576811, + "val_score": -2.6121638579153807 + }, + { + "epoch": 3, + "test_score": -2.5576997782221387, + "train_loss": 0.5093689668285954, + "val_score": -2.6009487768495023 + }, + { + "epoch": 4, + "test_score": -2.5850651586352034, + "train_loss": 0.5089051253623791, + "val_score": -2.6281073207918295 + }, + { + "epoch": 5, + "test_score": -2.5511833212856643, + "train_loss": 0.5059712389567951, + "val_score": -2.593261672781138 + }, + { + "epoch": 6, + "test_score": -2.551401433865984, + "train_loss": 0.5055853670781797, + "val_score": -2.5934982177598815 + }, + { + "epoch": 7, + "test_score": -2.5609792523241888, + "train_loss": 0.5017106358532433, + "val_score": -2.5997025539743244 + }, + { + "epoch": 8, + "test_score": -2.566694687611926, + "train_loss": 0.5020474761992961, + "val_score": -2.6070956936954204 + }, + { + "epoch": 9, + "test_score": -2.5532860134344815, + "train_loss": 0.5024598673120275, + "val_score": -2.5942565239541864 + }, + { + "epoch": 10, + "test_score": -2.579700982412669, + "train_loss": 0.499374769829415, + "val_score": -2.619462575902537 + }, + { + "epoch": 11, + "test_score": -2.557298172102745, + "train_loss": 0.4954140352773237, + "val_score": -2.597185364686641 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.5511833212856643, + "score": -2.5511833212856643 + }, + "val": { + "rmse": 2.593261672781138, + "score": -2.593261672781138 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 50.65602159500122, + "variant": "all_four_combined" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1.toml new file mode 100644 index 000000000..7dbf8b11c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1.toml @@ -0,0 +1,35 @@ +dataset = "cooking-time" +variant = "all_four_combined" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1/report.json new file mode 100644 index 000000000..3402f791d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1/report.json @@ -0,0 +1,106 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_four_combined" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.537508726809674, + "train_loss": 0.5776582470348289, + "val_score": -2.5844468721407705 + }, + { + "epoch": 1, + "test_score": -2.563174833433257, + "train_loss": 0.5161436847201338, + "val_score": -2.607712907300778 + }, + { + "epoch": 2, + "test_score": -2.5679362249447126, + "train_loss": 0.5124912057910953, + "val_score": -2.6130100902153788 + }, + { + "epoch": 3, + "test_score": -2.5667008183011366, + "train_loss": 0.509442943978954, + "val_score": -2.609481352506116 + }, + { + "epoch": 4, + "test_score": -2.581905206094061, + "train_loss": 0.5058386825226449, + "val_score": -2.6246502052852225 + }, + { + "epoch": 5, + "test_score": -2.558270476207951, + "train_loss": 0.5066947660467647, + "val_score": -2.599870010941221 + }, + { + "epoch": 6, + "test_score": -2.5907350965419256, + "train_loss": 0.504907029974568, + "val_score": -2.6330101776621766 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.537508726809674, + "score": -2.537508726809674 + }, + "val": { + "rmse": 2.5844468721407705, + "score": -2.5844468721407705 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 30.867638111114502, + "variant": "all_four_combined" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2.toml new file mode 100644 index 000000000..434c87c95 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2.toml @@ -0,0 +1,35 @@ +dataset = "cooking-time" +variant = "all_four_combined" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2/report.json new file mode 100644 index 000000000..9b3be73a4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2/report.json @@ -0,0 +1,118 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 2, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_four_combined" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.553360807438672, + "train_loss": 0.5795920593244536, + "val_score": -2.6008898349415337 + }, + { + "epoch": 1, + "test_score": -2.5549795001377738, + "train_loss": 0.5163620345227353, + "val_score": -2.600778914198739 + }, + { + "epoch": 2, + "test_score": -2.543946905739664, + "train_loss": 0.5128198091511253, + "val_score": -2.589426873028416 + }, + { + "epoch": 3, + "test_score": -2.55344026783191, + "train_loss": 0.5110959424091889, + "val_score": -2.5967557106208683 + }, + { + "epoch": 4, + "test_score": -2.550100612525452, + "train_loss": 0.5060882082393577, + "val_score": -2.5920808316036634 + }, + { + "epoch": 5, + "test_score": -2.584788271646388, + "train_loss": 0.5051669679783486, + "val_score": -2.624746855382455 + }, + { + "epoch": 6, + "test_score": -2.5773501994782806, + "train_loss": 0.5029631283906129, + "val_score": -2.6191585577138237 + }, + { + "epoch": 7, + "test_score": -2.5664566940854403, + "train_loss": 0.5020166579130534, + "val_score": -2.6072144843786274 + }, + { + "epoch": 8, + "test_score": -2.557857774486442, + "train_loss": 0.5009360981954111, + "val_score": -2.5982997470109916 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.543946905739664, + "score": -2.543946905739664 + }, + "val": { + "rmse": 2.589426873028416, + "score": -2.589426873028416 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 38.606435775756836, + "variant": "all_four_combined" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..d0a7f27f4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0.toml @@ -0,0 +1,34 @@ +dataset = "cooking-time" +variant = "all_minus_cf_fisd" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..8fd9c751e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0/report.json @@ -0,0 +1,117 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 2, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_minus_cf_fisd" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.597582914995142, + "train_loss": 0.5761679290114222, + "val_score": -2.6451580257118 + }, + { + "epoch": 1, + "test_score": -2.556929604770859, + "train_loss": 0.5164911712075139, + "val_score": -2.600416784626239 + }, + { + "epoch": 2, + "test_score": -2.546692730518933, + "train_loss": 0.5091461243930163, + "val_score": -2.5875247567139654 + }, + { + "epoch": 3, + "test_score": -2.5665785733087403, + "train_loss": 0.5083916611499615, + "val_score": -2.60850327601345 + }, + { + "epoch": 4, + "test_score": -2.592980420028019, + "train_loss": 0.5077804998234585, + "val_score": -2.634938997145239 + }, + { + "epoch": 5, + "test_score": -2.5491212639740826, + "train_loss": 0.5051470538517376, + "val_score": -2.591631196281184 + }, + { + "epoch": 6, + "test_score": -2.5484263370159503, + "train_loss": 0.5052224558752936, + "val_score": -2.5904392115991604 + }, + { + "epoch": 7, + "test_score": -2.5579466023184496, + "train_loss": 0.5017991039130065, + "val_score": -2.595014516683264 + }, + { + "epoch": 8, + "test_score": -2.5690880731651538, + "train_loss": 0.5029040357551059, + "val_score": -2.6090379143179177 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.546692730518933, + "score": -2.546692730518933 + }, + "val": { + "rmse": 2.5875247567139654, + "score": -2.5875247567139654 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 38.443965435028076, + "variant": "all_minus_cf_fisd" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1.toml new file mode 100644 index 000000000..ebfe27a78 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1.toml @@ -0,0 +1,34 @@ +dataset = "cooking-time" +variant = "all_minus_cf_fisd" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1/report.json new file mode 100644 index 000000000..0130e6f0f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1/report.json @@ -0,0 +1,105 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_minus_cf_fisd" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.5418282337637406, + "train_loss": 0.5805648884794734, + "val_score": -2.5864378108053976 + }, + { + "epoch": 1, + "test_score": -2.569724807106596, + "train_loss": 0.5154956329513241, + "val_score": -2.6111584810149067 + }, + { + "epoch": 2, + "test_score": -2.5603445338327986, + "train_loss": 0.511341643763018, + "val_score": -2.605412930143715 + }, + { + "epoch": 3, + "test_score": -2.5649989898858117, + "train_loss": 0.5074708193272084, + "val_score": -2.6060717112349616 + }, + { + "epoch": 4, + "test_score": -2.584029678097529, + "train_loss": 0.5050664694459589, + "val_score": -2.6266244675623907 + }, + { + "epoch": 5, + "test_score": -2.557123265415895, + "train_loss": 0.506016700385927, + "val_score": -2.599371184822983 + }, + { + "epoch": 6, + "test_score": -2.5861473344104584, + "train_loss": 0.5048232132250124, + "val_score": -2.628480692594635 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.5418282337637406, + "score": -2.5418282337637406 + }, + "val": { + "rmse": 2.5864378108053976, + "score": -2.5864378108053976 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 30.901761054992676, + "variant": "all_minus_cf_fisd" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2.toml new file mode 100644 index 000000000..1c796107d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2.toml @@ -0,0 +1,34 @@ +dataset = "cooking-time" +variant = "all_minus_cf_fisd" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2/report.json new file mode 100644 index 000000000..e41d048d7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2/report.json @@ -0,0 +1,111 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 1, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_minus_cf_fisd" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.5508004110414038, + "train_loss": 0.5819202207767211, + "val_score": -2.5966282696347784 + }, + { + "epoch": 1, + "test_score": -2.547009985219361, + "train_loss": 0.5161475149897842, + "val_score": -2.5906049665497832 + }, + { + "epoch": 2, + "test_score": -2.5508009718502036, + "train_loss": 0.5122214724888673, + "val_score": -2.5949883319981977 + }, + { + "epoch": 3, + "test_score": -2.564394366948711, + "train_loss": 0.5101304043520678, + "val_score": -2.607212929801097 + }, + { + "epoch": 4, + "test_score": -2.5620153015239624, + "train_loss": 0.505169359950332, + "val_score": -2.6036783498061484 + }, + { + "epoch": 5, + "test_score": -2.583565906597254, + "train_loss": 0.5044403819887487, + "val_score": -2.622261708127521 + }, + { + "epoch": 6, + "test_score": -2.5759304069394546, + "train_loss": 0.5026040581969528, + "val_score": -2.617307381232943 + }, + { + "epoch": 7, + "test_score": -2.568862738072886, + "train_loss": 0.5019908485648868, + "val_score": -2.607877838614469 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.547009985219361, + "score": -2.547009985219361 + }, + "val": { + "rmse": 2.5906049665497832, + "score": -2.5906049665497832 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 34.84113788604736, + "variant": "all_minus_cf_fisd" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0.toml new file mode 100644 index 000000000..7beafa65c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0.toml @@ -0,0 +1,33 @@ +dataset = "cooking-time" +variant = "all_minus_esam" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0/report.json new file mode 100644 index 000000000..581caaef2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0/report.json @@ -0,0 +1,182 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 13, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_minus_esam" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.5747772676005756, + "train_loss": 0.5732710495725408, + "val_score": -2.6218253420831923 + }, + { + "epoch": 1, + "test_score": -2.5564936514763854, + "train_loss": 0.5177499753934843, + "val_score": -2.6013183479183777 + }, + { + "epoch": 2, + "test_score": -2.5814181478565716, + "train_loss": 0.5109633602000572, + "val_score": -2.62519908331745 + }, + { + "epoch": 3, + "test_score": -2.574927827138447, + "train_loss": 0.5100945451775113, + "val_score": -2.6186524803194304 + }, + { + "epoch": 4, + "test_score": -2.5742446828644434, + "train_loss": 0.5061127572983235, + "val_score": -2.6172427954454816 + }, + { + "epoch": 5, + "test_score": -2.55020008798114, + "train_loss": 0.5048669292046143, + "val_score": -2.5901328913644526 + }, + { + "epoch": 6, + "test_score": -2.572474164916452, + "train_loss": 0.5023283319430308, + "val_score": -2.614022233998335 + }, + { + "epoch": 7, + "test_score": -2.5462834894134376, + "train_loss": 0.5025209985874795, + "val_score": -2.5865856637593847 + }, + { + "epoch": 8, + "test_score": -2.591484091195984, + "train_loss": 0.5010600809578423, + "val_score": -2.6324075902530604 + }, + { + "epoch": 9, + "test_score": -2.5581126918352166, + "train_loss": 0.5000964628683554, + "val_score": -2.598999593913042 + }, + { + "epoch": 10, + "test_score": -2.581940111176364, + "train_loss": 0.4989573072205793, + "val_score": -2.621668745437383 + }, + { + "epoch": 11, + "test_score": -2.554036842104361, + "train_loss": 0.49763295591414514, + "val_score": -2.594907571252727 + }, + { + "epoch": 12, + "test_score": -2.575861729251762, + "train_loss": 0.4963759712807767, + "val_score": -2.614391142530623 + }, + { + "epoch": 13, + "test_score": -2.5437589902976687, + "train_loss": 0.49609845984089485, + "val_score": -2.5833819805199907 + }, + { + "epoch": 14, + "test_score": -2.545113732695165, + "train_loss": 0.49491846615129764, + "val_score": -2.584010947992617 + }, + { + "epoch": 15, + "test_score": -2.5663227317377366, + "train_loss": 0.49232708306999895, + "val_score": -2.6043675968219655 + }, + { + "epoch": 16, + "test_score": -2.5473062349102906, + "train_loss": 0.4936659234064119, + "val_score": -2.585324125352541 + }, + { + "epoch": 17, + "test_score": -2.5794402495575657, + "train_loss": 0.4931852578579842, + "val_score": -2.618063162453346 + }, + { + "epoch": 18, + "test_score": -2.563614486616978, + "train_loss": 0.49165967402157484, + "val_score": -2.6031335434424254 + }, + { + "epoch": 19, + "test_score": -2.5898691581757034, + "train_loss": 0.4904036406461183, + "val_score": -2.6268180727366444 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.5437589902976687, + "score": -2.5437589902976687 + }, + "val": { + "rmse": 2.5833819805199907, + "score": -2.5833819805199907 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 58.42725872993469, + "variant": "all_minus_esam" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1.toml new file mode 100644 index 000000000..d712fef7c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1.toml @@ -0,0 +1,33 @@ +dataset = "cooking-time" +variant = "all_minus_esam" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1/report.json new file mode 100644 index 000000000..dc4288986 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1/report.json @@ -0,0 +1,134 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 5, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_minus_esam" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.546896343492815, + "train_loss": 0.5771154447181804, + "val_score": -2.5931625620194123 + }, + { + "epoch": 1, + "test_score": -2.563574588899563, + "train_loss": 0.5195186444230981, + "val_score": -2.608936935475554 + }, + { + "epoch": 2, + "test_score": -2.576534080926977, + "train_loss": 0.5107925359730248, + "val_score": -2.6210837442175823 + }, + { + "epoch": 3, + "test_score": -2.5532837723818225, + "train_loss": 0.5060298748918481, + "val_score": -2.5953448803998618 + }, + { + "epoch": 4, + "test_score": -2.566363144139515, + "train_loss": 0.5067531010052105, + "val_score": -2.6086219112022055 + }, + { + "epoch": 5, + "test_score": -2.538235385864352, + "train_loss": 0.5049025134460347, + "val_score": -2.5802569273242226 + }, + { + "epoch": 6, + "test_score": -2.5821721533391186, + "train_loss": 0.5031720842864063, + "val_score": -2.622583639522115 + }, + { + "epoch": 7, + "test_score": -2.573924856818587, + "train_loss": 0.5019908636002928, + "val_score": -2.6141672499897917 + }, + { + "epoch": 8, + "test_score": -2.5526273394139105, + "train_loss": 0.5017939341497851, + "val_score": -2.5910970149720978 + }, + { + "epoch": 9, + "test_score": -2.567319849634468, + "train_loss": 0.5008838630474366, + "val_score": -2.6068355058135366 + }, + { + "epoch": 10, + "test_score": -2.591484275197554, + "train_loss": 0.49883278210957843, + "val_score": -2.6313516882594596 + }, + { + "epoch": 11, + "test_score": -2.5877905714738745, + "train_loss": 0.49841721772073627, + "val_score": -2.6277731824390678 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.538235385864352, + "score": -2.538235385864352 + }, + "val": { + "rmse": 2.5802569273242226, + "score": -2.5802569273242226 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 37.65957021713257, + "variant": "all_minus_esam" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2.toml new file mode 100644 index 000000000..9c131ba6b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2.toml @@ -0,0 +1,33 @@ +dataset = "cooking-time" +variant = "all_minus_esam" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2/report.json new file mode 100644 index 000000000..fc4a1e67b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2/report.json @@ -0,0 +1,146 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 7, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_minus_esam" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.5613993630298912, + "train_loss": 0.5778566953298208, + "val_score": -2.6107928598405943 + }, + { + "epoch": 1, + "test_score": -2.550575797203022, + "train_loss": 0.5199141429888235, + "val_score": -2.5975772243324835 + }, + { + "epoch": 2, + "test_score": -2.5513964812165146, + "train_loss": 0.511967933661229, + "val_score": -2.5944791021852587 + }, + { + "epoch": 3, + "test_score": -2.5550999672998134, + "train_loss": 0.5085972109893421, + "val_score": -2.598609416731633 + }, + { + "epoch": 4, + "test_score": -2.5458052559244453, + "train_loss": 0.5063928671785303, + "val_score": -2.5876654530576864 + }, + { + "epoch": 5, + "test_score": -2.587354749942648, + "train_loss": 0.5060564223173503, + "val_score": -2.629049084709761 + }, + { + "epoch": 6, + "test_score": -2.566285756155162, + "train_loss": 0.5039662869126947, + "val_score": -2.607049236744262 + }, + { + "epoch": 7, + "test_score": -2.5476520499692263, + "train_loss": 0.5022442496574677, + "val_score": -2.586853602657961 + }, + { + "epoch": 8, + "test_score": -2.5592359828805464, + "train_loss": 0.5007378045503084, + "val_score": -2.599643675522678 + }, + { + "epoch": 9, + "test_score": -2.5641869365431083, + "train_loss": 0.49786975281732576, + "val_score": -2.6038341055557614 + }, + { + "epoch": 10, + "test_score": -2.580490599126014, + "train_loss": 0.4966755004616471, + "val_score": -2.6210794690079933 + }, + { + "epoch": 11, + "test_score": -2.571333291165615, + "train_loss": 0.49655182189769576, + "val_score": -2.6103619747617812 + }, + { + "epoch": 12, + "test_score": -2.5867859523181176, + "train_loss": 0.4945516618522438, + "val_score": -2.6269796263765466 + }, + { + "epoch": 13, + "test_score": -2.5806335267258453, + "train_loss": 0.49264940216734604, + "val_score": -2.620915641361488 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.5476520499692263, + "score": -2.5476520499692263 + }, + "val": { + "rmse": 2.586853602657961, + "score": -2.586853602657961 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 43.24603581428528, + "variant": "all_minus_esam" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0.toml new file mode 100644 index 000000000..73a24206b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0.toml @@ -0,0 +1,33 @@ +dataset = "cooking-time" +variant = "all_minus_mfb" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0/report.json new file mode 100644 index 000000000..1c3f751de --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0/report.json @@ -0,0 +1,122 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 3, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_minus_mfb" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.603120171403929, + "train_loss": 0.5561879529609336, + "val_score": -2.642832628894953 + }, + { + "epoch": 1, + "test_score": -2.588518681158689, + "train_loss": 0.4951903846886781, + "val_score": -2.626524527943006 + }, + { + "epoch": 2, + "test_score": -2.572048725550596, + "train_loss": 0.4900382980033084, + "val_score": -2.608899467218641 + }, + { + "epoch": 3, + "test_score": -2.572318549397406, + "train_loss": 0.4874864126647915, + "val_score": -2.60825858532094 + }, + { + "epoch": 4, + "test_score": -2.5802546172956813, + "train_loss": 0.4863069078943751, + "val_score": -2.6160648500237964 + }, + { + "epoch": 5, + "test_score": -2.5864119080151102, + "train_loss": 0.4848573691136128, + "val_score": -2.6218528046424394 + }, + { + "epoch": 6, + "test_score": -2.5831701678358927, + "train_loss": 0.48349400683566257, + "val_score": -2.617953424684935 + }, + { + "epoch": 7, + "test_score": -2.5753206664480017, + "train_loss": 0.48293743423513463, + "val_score": -2.6101944601761153 + }, + { + "epoch": 8, + "test_score": -2.5976935133018086, + "train_loss": 0.4819742309617567, + "val_score": -2.6332332829490097 + }, + { + "epoch": 9, + "test_score": -2.5914011971600104, + "train_loss": 0.4817004324616613, + "val_score": -2.623775287304586 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.572318549397406, + "score": -2.572318549397406 + }, + "val": { + "rmse": 2.60825858532094, + "score": -2.60825858532094 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 42.692240953445435, + "variant": "all_minus_mfb" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1.toml new file mode 100644 index 000000000..2cbd4e61c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1.toml @@ -0,0 +1,33 @@ +dataset = "cooking-time" +variant = "all_minus_mfb" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1/report.json new file mode 100644 index 000000000..9230108cd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1/report.json @@ -0,0 +1,122 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 3, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_minus_mfb" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.561574350040879, + "train_loss": 0.5604447545232, + "val_score": -2.6029467866304032 + }, + { + "epoch": 1, + "test_score": -2.574204301576486, + "train_loss": 0.4954050814246272, + "val_score": -2.6123855492741956 + }, + { + "epoch": 2, + "test_score": -2.5578780010321176, + "train_loss": 0.49020274399637104, + "val_score": -2.5961486571822516 + }, + { + "epoch": 3, + "test_score": -2.5472973432410306, + "train_loss": 0.48780881499384976, + "val_score": -2.5833460797231274 + }, + { + "epoch": 4, + "test_score": -2.593680599157413, + "train_loss": 0.4862094964530017, + "val_score": -2.62832068260243 + }, + { + "epoch": 5, + "test_score": -2.569472340841902, + "train_loss": 0.4844946855897302, + "val_score": -2.605646267495547 + }, + { + "epoch": 6, + "test_score": -2.5842775854642452, + "train_loss": 0.4840661797437582, + "val_score": -2.618392530274335 + }, + { + "epoch": 7, + "test_score": -2.5719627949410135, + "train_loss": 0.48236743233225365, + "val_score": -2.6085260346458674 + }, + { + "epoch": 8, + "test_score": -2.561136580823507, + "train_loss": 0.48235606825029526, + "val_score": -2.5962453581502647 + }, + { + "epoch": 9, + "test_score": -2.57570807712001, + "train_loss": 0.48123758950748957, + "val_score": -2.611934571473405 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.5472973432410306, + "score": -2.5472973432410306 + }, + "val": { + "rmse": 2.5833460797231274, + "score": -2.5833460797231274 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 44.12882924079895, + "variant": "all_minus_mfb" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2.toml new file mode 100644 index 000000000..990e8d379 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2.toml @@ -0,0 +1,33 @@ +dataset = "cooking-time" +variant = "all_minus_mfb" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2/report.json new file mode 100644 index 000000000..f593ca0d5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2/report.json @@ -0,0 +1,110 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 1, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_minus_mfb" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.603077673510457, + "train_loss": 0.5612956862728875, + "val_score": -2.643770770629382 + }, + { + "epoch": 1, + "test_score": -2.5601664827274426, + "train_loss": 0.4957994515294427, + "val_score": -2.599479505818785 + }, + { + "epoch": 2, + "test_score": -2.5856204112203303, + "train_loss": 0.4903566123129011, + "val_score": -2.6224372706443058 + }, + { + "epoch": 3, + "test_score": -2.5785627513659493, + "train_loss": 0.4880300644818727, + "val_score": -2.6154062159108005 + }, + { + "epoch": 4, + "test_score": -2.5974606547895833, + "train_loss": 0.48578819578832333, + "val_score": -2.63218314688471 + }, + { + "epoch": 5, + "test_score": -2.5809724746294607, + "train_loss": 0.4853922931460647, + "val_score": -2.615697727081015 + }, + { + "epoch": 6, + "test_score": -2.5823524725431946, + "train_loss": 0.4840138941197782, + "val_score": -2.6170042970219027 + }, + { + "epoch": 7, + "test_score": -2.568776700885788, + "train_loss": 0.48290939508257685, + "val_score": -2.6039036020287787 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.5601664827274426, + "score": -2.5601664827274426 + }, + "val": { + "rmse": 2.599479505818785, + "score": -2.599479505818785 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 35.43584752082825, + "variant": "all_minus_mfb" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0.toml new file mode 100644 index 000000000..c0c763e13 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0.toml @@ -0,0 +1,32 @@ +dataset = "cooking-time" +variant = "all_minus_rla" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0/report.json new file mode 100644 index 000000000..aafa0dc07 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0/report.json @@ -0,0 +1,139 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 6, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_minus_rla" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.5764028635071594, + "train_loss": 0.573144529048387, + "val_score": -2.6233867047352715 + }, + { + "epoch": 1, + "test_score": -2.5481321824369987, + "train_loss": 0.518380832296234, + "val_score": -2.593426328019743 + }, + { + "epoch": 2, + "test_score": -2.568308596961209, + "train_loss": 0.5127465303416725, + "val_score": -2.6122966559455976 + }, + { + "epoch": 3, + "test_score": -2.56838332480516, + "train_loss": 0.5098949592929702, + "val_score": -2.6127086975845275 + }, + { + "epoch": 4, + "test_score": -2.5867971967751267, + "train_loss": 0.5094581716232471, + "val_score": -2.6301702716956505 + }, + { + "epoch": 5, + "test_score": -2.557593136878615, + "train_loss": 0.5076963931053609, + "val_score": -2.5998900940480447 + }, + { + "epoch": 6, + "test_score": -2.550683292768776, + "train_loss": 0.5074400810508041, + "val_score": -2.5932213118056806 + }, + { + "epoch": 7, + "test_score": -2.5802537856849, + "train_loss": 0.5039221112792557, + "val_score": -2.6213640735928387 + }, + { + "epoch": 8, + "test_score": -2.5604579511458847, + "train_loss": 0.5042530888909692, + "val_score": -2.601104512726605 + }, + { + "epoch": 9, + "test_score": -2.5711370844102874, + "train_loss": 0.5058108028527852, + "val_score": -2.6124406725704215 + }, + { + "epoch": 10, + "test_score": -2.582553089404887, + "train_loss": 0.5030083006566709, + "val_score": -2.6239106779073076 + }, + { + "epoch": 11, + "test_score": -2.575217902583891, + "train_loss": 0.49936754934422606, + "val_score": -2.616333961798112 + }, + { + "epoch": 12, + "test_score": -2.5628126000259095, + "train_loss": 0.5023714462379077, + "val_score": -2.6039989163093247 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.550683292768776, + "score": -2.550683292768776 + }, + "val": { + "rmse": 2.5932213118056806, + "score": -2.5932213118056806 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 49.47507905960083, + "variant": "all_minus_rla" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1.toml new file mode 100644 index 000000000..05d224989 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1.toml @@ -0,0 +1,32 @@ +dataset = "cooking-time" +variant = "all_minus_rla" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1/report.json new file mode 100644 index 000000000..b6f3ebac3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1/report.json @@ -0,0 +1,103 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_minus_rla" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.5451569174185407, + "train_loss": 0.5805180867513021, + "val_score": -2.5924488158953323 + }, + { + "epoch": 1, + "test_score": -2.5729383604136413, + "train_loss": 0.5178044803507693, + "val_score": -2.618478940350594 + }, + { + "epoch": 2, + "test_score": -2.573552463652647, + "train_loss": 0.5117641367890813, + "val_score": -2.617134207567258 + }, + { + "epoch": 3, + "test_score": -2.566955414750782, + "train_loss": 0.5101196943102656, + "val_score": -2.6105366022963 + }, + { + "epoch": 4, + "test_score": -2.573111821442705, + "train_loss": 0.5074310834343368, + "val_score": -2.617440100479082 + }, + { + "epoch": 5, + "test_score": -2.5662011190851253, + "train_loss": 0.5082239653613116, + "val_score": -2.6090179016271127 + }, + { + "epoch": 6, + "test_score": -2.5780162441549805, + "train_loss": 0.5056800353634465, + "val_score": -2.6210339876232287 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.5451569174185407, + "score": -2.5451569174185407 + }, + "val": { + "rmse": 2.5924488158953323, + "score": -2.5924488158953323 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 26.603979349136353, + "variant": "all_minus_rla" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2.toml new file mode 100644 index 000000000..e5fdb6826 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2.toml @@ -0,0 +1,32 @@ +dataset = "cooking-time" +variant = "all_minus_rla" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2/report.json new file mode 100644 index 000000000..7e716b47b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2/report.json @@ -0,0 +1,157 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 9, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_minus_rla" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.564628926255888, + "train_loss": 0.5797603635100631, + "val_score": -2.6127031311205617 + }, + { + "epoch": 1, + "test_score": -2.567705031902881, + "train_loss": 0.5178446670373281, + "val_score": -2.6129844508389115 + }, + { + "epoch": 2, + "test_score": -2.562137019803166, + "train_loss": 0.5133768392575754, + "val_score": -2.607213752812846 + }, + { + "epoch": 3, + "test_score": -2.584532756672374, + "train_loss": 0.5111699088199718, + "val_score": -2.6288309752526415 + }, + { + "epoch": 4, + "test_score": -2.5562409044024648, + "train_loss": 0.5082123596389014, + "val_score": -2.5991978251489503 + }, + { + "epoch": 5, + "test_score": -2.5670301819867083, + "train_loss": 0.5054270923137665, + "val_score": -2.6091194256096264 + }, + { + "epoch": 6, + "test_score": -2.5778272977088172, + "train_loss": 0.5041925184898548, + "val_score": -2.6198486463494515 + }, + { + "epoch": 7, + "test_score": -2.5606990160869687, + "train_loss": 0.5048417513434952, + "val_score": -2.602276220164088 + }, + { + "epoch": 8, + "test_score": -2.5618564448879386, + "train_loss": 0.5054062043761348, + "val_score": -2.603846832980074 + }, + { + "epoch": 9, + "test_score": -2.544944827487872, + "train_loss": 0.5027262665129997, + "val_score": -2.58848699643732 + }, + { + "epoch": 10, + "test_score": -2.5682558684288046, + "train_loss": 0.5023538174929919, + "val_score": -2.608500351194885 + }, + { + "epoch": 11, + "test_score": -2.58327123095612, + "train_loss": 0.5022286219639821, + "val_score": -2.624788184929799 + }, + { + "epoch": 12, + "test_score": -2.5471878327692514, + "train_loss": 0.5031755037672885, + "val_score": -2.588843058815793 + }, + { + "epoch": 13, + "test_score": -2.560687470813483, + "train_loss": 0.5017490123843288, + "val_score": -2.602519824330865 + }, + { + "epoch": 14, + "test_score": -2.5609168768116946, + "train_loss": 0.503594289223353, + "val_score": -2.600978201470113 + }, + { + "epoch": 15, + "test_score": -2.570908590376566, + "train_loss": 0.5005721874065228, + "val_score": -2.6146657153116015 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.544944827487872, + "score": -2.544944827487872 + }, + "val": { + "rmse": 2.58848699643732, + "score": -2.58848699643732 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 62.630138874053955, + "variant": "all_minus_rla" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..8894099eb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0.toml @@ -0,0 +1,27 @@ +dataset = "cooking-time" +variant = "baseline_plr" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..e1a2faf38 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0/report.json @@ -0,0 +1,116 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 3, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "baseline_plr" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.5598227308409616, + "train_loss": 0.562614095640612, + "val_score": -2.600956935110999 + }, + { + "epoch": 1, + "test_score": -2.56268421534399, + "train_loss": 0.49585827218519674, + "val_score": -2.6005418396268225 + }, + { + "epoch": 2, + "test_score": -2.5828534777852807, + "train_loss": 0.4898477091982558, + "val_score": -2.6206401767923544 + }, + { + "epoch": 3, + "test_score": -2.5612615058011223, + "train_loss": 0.48662275988776404, + "val_score": -2.5972627499478844 + }, + { + "epoch": 4, + "test_score": -2.6042154434722344, + "train_loss": 0.48567526619713586, + "val_score": -2.641161350652755 + }, + { + "epoch": 5, + "test_score": -2.5837059880214395, + "train_loss": 0.4844431109256572, + "val_score": -2.617444381633043 + }, + { + "epoch": 6, + "test_score": -2.5854192946362007, + "train_loss": 0.4830809208723876, + "val_score": -2.61894372118841 + }, + { + "epoch": 7, + "test_score": -2.5704376283004158, + "train_loss": 0.4821285430912499, + "val_score": -2.604933287632126 + }, + { + "epoch": 8, + "test_score": -2.597135425642725, + "train_loss": 0.4808750023712983, + "val_score": -2.6334981959278654 + }, + { + "epoch": 9, + "test_score": -2.578672223825131, + "train_loss": 0.4806423933656366, + "val_score": -2.612789181852867 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.5612615058011223, + "score": -2.5612615058011223 + }, + "val": { + "rmse": 2.5972627499478844, + "score": -2.5972627499478844 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 25.95792293548584, + "variant": "baseline_plr" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1.toml new file mode 100644 index 000000000..ca4c2e8a1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1.toml @@ -0,0 +1,27 @@ +dataset = "cooking-time" +variant = "baseline_plr" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1/report.json new file mode 100644 index 000000000..ee4ef6298 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1/report.json @@ -0,0 +1,116 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 3, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "baseline_plr" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.585175370220307, + "train_loss": 0.5666897385507017, + "val_score": -2.6246198651410304 + }, + { + "epoch": 1, + "test_score": -2.5758223914764855, + "train_loss": 0.4957716354915688, + "val_score": -2.6152098514241806 + }, + { + "epoch": 2, + "test_score": -2.5674024068230503, + "train_loss": 0.48955290032936644, + "val_score": -2.6025422688476967 + }, + { + "epoch": 3, + "test_score": -2.5657847681143955, + "train_loss": 0.48720274019885706, + "val_score": -2.601873705696629 + }, + { + "epoch": 4, + "test_score": -2.5715011120963838, + "train_loss": 0.48514884549218257, + "val_score": -2.607710804450608 + }, + { + "epoch": 5, + "test_score": -2.5812347153033297, + "train_loss": 0.48385220712369625, + "val_score": -2.6151459431814095 + }, + { + "epoch": 6, + "test_score": -2.5934737644401955, + "train_loss": 0.4826641380786896, + "val_score": -2.6289346361824175 + }, + { + "epoch": 7, + "test_score": -2.5747132817609746, + "train_loss": 0.48209835602356504, + "val_score": -2.607563966458419 + }, + { + "epoch": 8, + "test_score": -2.574277283809098, + "train_loss": 0.48115245984481264, + "val_score": -2.6069413217336055 + }, + { + "epoch": 9, + "test_score": -2.5774853461928484, + "train_loss": 0.4803670027234533, + "val_score": -2.6114248116625056 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.5657847681143955, + "score": -2.5657847681143955 + }, + "val": { + "rmse": 2.601873705696629, + "score": -2.601873705696629 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 28.450456857681274, + "variant": "baseline_plr" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2.toml new file mode 100644 index 000000000..21f772c82 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2.toml @@ -0,0 +1,27 @@ +dataset = "cooking-time" +variant = "baseline_plr" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2/report.json new file mode 100644 index 000000000..57677369c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2/report.json @@ -0,0 +1,158 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 10, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "baseline_plr" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.5753881550792515, + "train_loss": 0.566168913701633, + "val_score": -2.6158977917760935 + }, + { + "epoch": 1, + "test_score": -2.574276265035827, + "train_loss": 0.4963748224146731, + "val_score": -2.612004217532902 + }, + { + "epoch": 2, + "test_score": -2.5944422521951775, + "train_loss": 0.4892988766098882, + "val_score": -2.630207799527457 + }, + { + "epoch": 3, + "test_score": -2.5944665125966364, + "train_loss": 0.4868296041144981, + "val_score": -2.631199826738509 + }, + { + "epoch": 4, + "test_score": -2.5889661864263944, + "train_loss": 0.48513829654401486, + "val_score": -2.625051770380953 + }, + { + "epoch": 5, + "test_score": -2.5700625034257225, + "train_loss": 0.4835451285581331, + "val_score": -2.6054604228514324 + }, + { + "epoch": 6, + "test_score": -2.5928133456792115, + "train_loss": 0.4826182731100031, + "val_score": -2.6278055729470218 + }, + { + "epoch": 7, + "test_score": -2.5798912702611383, + "train_loss": 0.4817298229213233, + "val_score": -2.6160575591063306 + }, + { + "epoch": 8, + "test_score": -2.5656927734047232, + "train_loss": 0.4806378687824215, + "val_score": -2.6017324947685756 + }, + { + "epoch": 9, + "test_score": -2.5914622869174426, + "train_loss": 0.4803790836720853, + "val_score": -2.62708953162926 + }, + { + "epoch": 10, + "test_score": -2.5666563240272247, + "train_loss": 0.4802004740581856, + "val_score": -2.6010690398654326 + }, + { + "epoch": 11, + "test_score": -2.5836036499682753, + "train_loss": 0.4790540662434724, + "val_score": -2.6167414498627273 + }, + { + "epoch": 12, + "test_score": -2.595542288476678, + "train_loss": 0.4786413835512625, + "val_score": -2.6289079731316325 + }, + { + "epoch": 13, + "test_score": -2.5878016273022295, + "train_loss": 0.47891669821094823, + "val_score": -2.622896441522318 + }, + { + "epoch": 14, + "test_score": -2.585362119711485, + "train_loss": 0.4776747846925581, + "val_score": -2.619510086934299 + }, + { + "epoch": 15, + "test_score": -2.5852033143357676, + "train_loss": 0.47722361324069734, + "val_score": -2.6188749879858246 + }, + { + "epoch": 16, + "test_score": -2.597851371607391, + "train_loss": 0.47689444954330856, + "val_score": -2.6305574896685715 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.5666563240272247, + "score": -2.5666563240272247 + }, + "val": { + "rmse": 2.6010690398654326, + "score": -2.6010690398654326 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 49.608482360839844, + "variant": "baseline_plr" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..d16fef85d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0.toml @@ -0,0 +1,28 @@ +dataset = "cooking-time" +variant = "cf_fisd_only" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..b07b4dfc0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,99 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "cf_fisd_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.563118185514718, + "train_loss": 0.5551990426338471, + "val_score": -2.6028544565868588 + }, + { + "epoch": 1, + "test_score": -2.572042144127544, + "train_loss": 0.4948291037533734, + "val_score": -2.6100162471981716 + }, + { + "epoch": 2, + "test_score": -2.5773333634568774, + "train_loss": 0.489956555065808, + "val_score": -2.6151421141072886 + }, + { + "epoch": 3, + "test_score": -2.566805130662705, + "train_loss": 0.48730920859285304, + "val_score": -2.6032010437115134 + }, + { + "epoch": 4, + "test_score": -2.610084665613344, + "train_loss": 0.48609176707697344, + "val_score": -2.6468524522057435 + }, + { + "epoch": 5, + "test_score": -2.5852612305505973, + "train_loss": 0.48481034373377896, + "val_score": -2.619681374343505 + }, + { + "epoch": 6, + "test_score": -2.586320554822736, + "train_loss": 0.4835801567580249, + "val_score": -2.620107177988242 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.563118185514718, + "score": -2.563118185514718 + }, + "val": { + "rmse": 2.6028544565868588, + "score": -2.6028544565868588 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 18.744648456573486, + "variant": "cf_fisd_only" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1.toml new file mode 100644 index 000000000..1e11ee101 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1.toml @@ -0,0 +1,28 @@ +dataset = "cooking-time" +variant = "cf_fisd_only" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1/report.json new file mode 100644 index 000000000..98da7e105 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1/report.json @@ -0,0 +1,117 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 3, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "cf_fisd_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.5768438681813626, + "train_loss": 0.5615193841156659, + "val_score": -2.6179108032989857 + }, + { + "epoch": 1, + "test_score": -2.5817535758940755, + "train_loss": 0.49551311296385686, + "val_score": -2.6201837952771014 + }, + { + "epoch": 2, + "test_score": -2.5668466500759455, + "train_loss": 0.4899280680729462, + "val_score": -2.6034090287864857 + }, + { + "epoch": 3, + "test_score": -2.5605115851699063, + "train_loss": 0.4877959296510026, + "val_score": -2.5968646915749916 + }, + { + "epoch": 4, + "test_score": -2.572959951018651, + "train_loss": 0.48579797497740734, + "val_score": -2.6096605156120183 + }, + { + "epoch": 5, + "test_score": -2.584487093393811, + "train_loss": 0.48443509812827584, + "val_score": -2.618533662222361 + }, + { + "epoch": 6, + "test_score": -2.598905655830961, + "train_loss": 0.48327650465406813, + "val_score": -2.634865342523215 + }, + { + "epoch": 7, + "test_score": -2.5753543646961994, + "train_loss": 0.48260759662937475, + "val_score": -2.6092616987471433 + }, + { + "epoch": 8, + "test_score": -2.5748974566735727, + "train_loss": 0.48181808666066006, + "val_score": -2.6095718949093794 + }, + { + "epoch": 9, + "test_score": -2.5797974681462987, + "train_loss": 0.48100584772256044, + "val_score": -2.614384850089723 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.5605115851699063, + "score": -2.5605115851699063 + }, + "val": { + "rmse": 2.5968646915749916, + "score": -2.5968646915749916 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 28.90973997116089, + "variant": "cf_fisd_only" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2.toml new file mode 100644 index 000000000..850658dc6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2.toml @@ -0,0 +1,28 @@ +dataset = "cooking-time" +variant = "cf_fisd_only" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2/report.json new file mode 100644 index 000000000..b164f8c4c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2/report.json @@ -0,0 +1,159 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 10, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "cf_fisd_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.5875577431837735, + "train_loss": 0.5611629059185853, + "val_score": -2.6295552465826466 + }, + { + "epoch": 1, + "test_score": -2.5747327276980787, + "train_loss": 0.4955460289577106, + "val_score": -2.6127804217806334 + }, + { + "epoch": 2, + "test_score": -2.598434996933457, + "train_loss": 0.489554179681314, + "val_score": -2.6344989389970186 + }, + { + "epoch": 3, + "test_score": -2.593255788760048, + "train_loss": 0.48722675120508346, + "val_score": -2.630046806789206 + }, + { + "epoch": 4, + "test_score": -2.591796323061829, + "train_loss": 0.485682710065498, + "val_score": -2.6285540726041305 + }, + { + "epoch": 5, + "test_score": -2.57461521644028, + "train_loss": 0.4841639764674075, + "val_score": -2.6101172756287023 + }, + { + "epoch": 6, + "test_score": -2.5907818460322876, + "train_loss": 0.4832287656294333, + "val_score": -2.6265027422350546 + }, + { + "epoch": 7, + "test_score": -2.5792667519455312, + "train_loss": 0.4822454852563841, + "val_score": -2.613920079539629 + }, + { + "epoch": 8, + "test_score": -2.567953401107109, + "train_loss": 0.4813956851894791, + "val_score": -2.6037318260814915 + }, + { + "epoch": 9, + "test_score": -2.5921374904230277, + "train_loss": 0.4810924151459256, + "val_score": -2.6279573584396494 + }, + { + "epoch": 10, + "test_score": -2.566079036110905, + "train_loss": 0.48065090662724264, + "val_score": -2.600480321386129 + }, + { + "epoch": 11, + "test_score": -2.5830334725214104, + "train_loss": 0.4798199037710826, + "val_score": -2.6167372586716477 + }, + { + "epoch": 12, + "test_score": -2.594220681743779, + "train_loss": 0.4794894346782753, + "val_score": -2.6278059358636168 + }, + { + "epoch": 13, + "test_score": -2.5928668621350965, + "train_loss": 0.4797732456847354, + "val_score": -2.628311702161112 + }, + { + "epoch": 14, + "test_score": -2.59032379350422, + "train_loss": 0.47852054238319397, + "val_score": -2.6244594380693966 + }, + { + "epoch": 15, + "test_score": -2.585993740242376, + "train_loss": 0.4782114834398837, + "val_score": -2.620114912614322 + }, + { + "epoch": 16, + "test_score": -2.595159492368922, + "train_loss": 0.4778157437706853, + "val_score": -2.629266541322618 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.566079036110905, + "score": -2.566079036110905 + }, + "val": { + "rmse": 2.600480321386129, + "score": -2.600480321386129 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 46.023908615112305, + "variant": "cf_fisd_only" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0.toml new file mode 100644 index 000000000..8afcd8fb2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0.toml @@ -0,0 +1,29 @@ +dataset = "cooking-time" +variant = "esam_only" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0/report.json new file mode 100644 index 000000000..ba2b2b1de --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0/report.json @@ -0,0 +1,118 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 3, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "esam_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.5576536358788435, + "train_loss": 0.5631172922817437, + "val_score": -2.598798228335147 + }, + { + "epoch": 1, + "test_score": -2.5601839903934147, + "train_loss": 0.49629482143634074, + "val_score": -2.5981515512584874 + }, + { + "epoch": 2, + "test_score": -2.584350375481377, + "train_loss": 0.49001724709261646, + "val_score": -2.6219641069040334 + }, + { + "epoch": 3, + "test_score": -2.5611478448009337, + "train_loss": 0.48687619266209303, + "val_score": -2.5972630253361557 + }, + { + "epoch": 4, + "test_score": -2.6040067903397914, + "train_loss": 0.485992630859753, + "val_score": -2.6409654567124847 + }, + { + "epoch": 5, + "test_score": -2.5828276313573895, + "train_loss": 0.4846961060622791, + "val_score": -2.6166486042962434 + }, + { + "epoch": 6, + "test_score": -2.58425830360799, + "train_loss": 0.4833594560623169, + "val_score": -2.6178357588304086 + }, + { + "epoch": 7, + "test_score": -2.56968583929098, + "train_loss": 0.4823442751222903, + "val_score": -2.6042412607317895 + }, + { + "epoch": 8, + "test_score": -2.5962759380486657, + "train_loss": 0.48112203865437897, + "val_score": -2.6326239543833205 + }, + { + "epoch": 9, + "test_score": -2.577815274216168, + "train_loss": 0.48088436835520976, + "val_score": -2.6120958590596373 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.5611478448009337, + "score": -2.5611478448009337 + }, + "val": { + "rmse": 2.5972630253361557, + "score": -2.5972630253361557 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 36.826345682144165, + "variant": "esam_only" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1.toml new file mode 100644 index 000000000..33b908233 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1.toml @@ -0,0 +1,29 @@ +dataset = "cooking-time" +variant = "esam_only" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1/report.json new file mode 100644 index 000000000..27b344757 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1/report.json @@ -0,0 +1,118 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 3, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "esam_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.587351709069442, + "train_loss": 0.5673266920420501, + "val_score": -2.6270270922149246 + }, + { + "epoch": 1, + "test_score": -2.574544003304502, + "train_loss": 0.4962879950935776, + "val_score": -2.614041752342434 + }, + { + "epoch": 2, + "test_score": -2.567338701509846, + "train_loss": 0.48992319713841687, + "val_score": -2.602752321820629 + }, + { + "epoch": 3, + "test_score": -2.563722179453466, + "train_loss": 0.4875023689892915, + "val_score": -2.600100452976782 + }, + { + "epoch": 4, + "test_score": -2.5729477194535098, + "train_loss": 0.4853253735078348, + "val_score": -2.609250368351265 + }, + { + "epoch": 5, + "test_score": -2.5810642018541534, + "train_loss": 0.48425327791823997, + "val_score": -2.6150257805160684 + }, + { + "epoch": 6, + "test_score": -2.594595989463728, + "train_loss": 0.4828814523177104, + "val_score": -2.6301171516753516 + }, + { + "epoch": 7, + "test_score": -2.5756927114027013, + "train_loss": 0.4822785379113378, + "val_score": -2.6085275884408397 + }, + { + "epoch": 8, + "test_score": -2.572590569179888, + "train_loss": 0.4814685883822742, + "val_score": -2.6054338856070185 + }, + { + "epoch": 9, + "test_score": -2.576577479294077, + "train_loss": 0.48072265773206146, + "val_score": -2.6107369711738118 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.563722179453466, + "score": -2.563722179453466 + }, + "val": { + "rmse": 2.600100452976782, + "score": -2.600100452976782 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 39.21153426170349, + "variant": "esam_only" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2.toml new file mode 100644 index 000000000..f6e5c3aa3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2.toml @@ -0,0 +1,29 @@ +dataset = "cooking-time" +variant = "esam_only" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2/report.json new file mode 100644 index 000000000..42dccb37a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2/report.json @@ -0,0 +1,160 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 10, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "esam_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.5730321346015366, + "train_loss": 0.566731651087065, + "val_score": -2.6135744577547486 + }, + { + "epoch": 1, + "test_score": -2.5750931917382087, + "train_loss": 0.49662325430560755, + "val_score": -2.61291455716844 + }, + { + "epoch": 2, + "test_score": -2.593852029655047, + "train_loss": 0.48959457659506583, + "val_score": -2.629839840497881 + }, + { + "epoch": 3, + "test_score": -2.59720170482707, + "train_loss": 0.48712443526800686, + "val_score": -2.6338706224915773 + }, + { + "epoch": 4, + "test_score": -2.5891744862235693, + "train_loss": 0.4853177621020927, + "val_score": -2.625148496514752 + }, + { + "epoch": 5, + "test_score": -2.5708206741128494, + "train_loss": 0.48389967470555695, + "val_score": -2.606276265476257 + }, + { + "epoch": 6, + "test_score": -2.5933733747115633, + "train_loss": 0.4827877854978716, + "val_score": -2.6281835234268742 + }, + { + "epoch": 7, + "test_score": -2.5794014285161495, + "train_loss": 0.4820292495392464, + "val_score": -2.615673754749277 + }, + { + "epoch": 8, + "test_score": -2.5661116478848696, + "train_loss": 0.48090581152890177, + "val_score": -2.60218120927395 + }, + { + "epoch": 9, + "test_score": -2.5917589749725156, + "train_loss": 0.48059917328593965, + "val_score": -2.627395172931714 + }, + { + "epoch": 10, + "test_score": -2.56671335830164, + "train_loss": 0.4804097650287388, + "val_score": -2.601435477790095 + }, + { + "epoch": 11, + "test_score": -2.584341611265953, + "train_loss": 0.4794119491770461, + "val_score": -2.6174121361735105 + }, + { + "epoch": 12, + "test_score": -2.595533745766955, + "train_loss": 0.4789287713733879, + "val_score": -2.6290037411981997 + }, + { + "epoch": 13, + "test_score": -2.587051842988804, + "train_loss": 0.47913944774919803, + "val_score": -2.622268709032895 + }, + { + "epoch": 14, + "test_score": -2.585114869714288, + "train_loss": 0.47785790546520335, + "val_score": -2.6195328409546783 + }, + { + "epoch": 15, + "test_score": -2.5854203090186108, + "train_loss": 0.47741706763301883, + "val_score": -2.619167751595635 + }, + { + "epoch": 16, + "test_score": -2.598019039662804, + "train_loss": 0.47704946135615445, + "val_score": -2.630867440509398 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.56671335830164, + "score": -2.56671335830164 + }, + "val": { + "rmse": 2.601435477790095, + "score": -2.601435477790095 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 66.56187295913696, + "variant": "esam_only" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0.toml new file mode 100644 index 000000000..ffa62de37 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0.toml @@ -0,0 +1,29 @@ +dataset = "cooking-time" +variant = "mfb_only" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..f8cefbf30 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0/report.json @@ -0,0 +1,130 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 5, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "mfb_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.5661236333044846, + "train_loss": 0.5758807992076015, + "val_score": -2.6119257172541515 + }, + { + "epoch": 1, + "test_score": -2.5743387799216704, + "train_loss": 0.5163963157851417, + "val_score": -2.618051870148412 + }, + { + "epoch": 2, + "test_score": -2.5601411522753925, + "train_loss": 0.5085933527967952, + "val_score": -2.6044778154775474 + }, + { + "epoch": 3, + "test_score": -2.576163730796452, + "train_loss": 0.5093893889371339, + "val_score": -2.619102301377666 + }, + { + "epoch": 4, + "test_score": -2.57161255397797, + "train_loss": 0.505496577905105, + "val_score": -2.6141280326221903 + }, + { + "epoch": 5, + "test_score": -2.5555160998555837, + "train_loss": 0.5060446619450509, + "val_score": -2.5961102696644223 + }, + { + "epoch": 6, + "test_score": -2.5676436554743733, + "train_loss": 0.5035226178598834, + "val_score": -2.6076490896143696 + }, + { + "epoch": 7, + "test_score": -2.5625663376571928, + "train_loss": 0.5044513223944483, + "val_score": -2.6047008017490376 + }, + { + "epoch": 8, + "test_score": -2.583804446506284, + "train_loss": 0.50434070288598, + "val_score": -2.6261775690664138 + }, + { + "epoch": 9, + "test_score": -2.585765452582473, + "train_loss": 0.5042245648465715, + "val_score": -2.627743604203495 + }, + { + "epoch": 10, + "test_score": -2.5881280293524083, + "train_loss": 0.5026848740405865, + "val_score": -2.6263449720071628 + }, + { + "epoch": 11, + "test_score": -2.567627870074032, + "train_loss": 0.501694805718757, + "val_score": -2.6076158087112957 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.5555160998555837, + "score": -2.5555160998555837 + }, + "val": { + "rmse": 2.5961102696644223, + "score": -2.5961102696644223 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 30.617911338806152, + "variant": "mfb_only" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1.toml new file mode 100644 index 000000000..c953cfa7f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1.toml @@ -0,0 +1,29 @@ +dataset = "cooking-time" +variant = "mfb_only" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1/report.json new file mode 100644 index 000000000..3ba3c8265 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1/report.json @@ -0,0 +1,142 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 7, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "mfb_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.5797754726740076, + "train_loss": 0.5817791330921758, + "val_score": -2.623108227566879 + }, + { + "epoch": 1, + "test_score": -2.5741110331989514, + "train_loss": 0.518071799664884, + "val_score": -2.618382423113911 + }, + { + "epoch": 2, + "test_score": -2.572947534126318, + "train_loss": 0.5103391008871095, + "val_score": -2.6161554379786542 + }, + { + "epoch": 3, + "test_score": -2.558461426086861, + "train_loss": 0.506716252715738, + "val_score": -2.6023681126430156 + }, + { + "epoch": 4, + "test_score": -2.5548843167984927, + "train_loss": 0.5062507541330011, + "val_score": -2.5978860624377282 + }, + { + "epoch": 5, + "test_score": -2.5596182832148666, + "train_loss": 0.5055183303785754, + "val_score": -2.6004166929414816 + }, + { + "epoch": 6, + "test_score": -2.561558620314518, + "train_loss": 0.5038739445510211, + "val_score": -2.603257826842896 + }, + { + "epoch": 7, + "test_score": -2.539276959092014, + "train_loss": 0.5045690974136731, + "val_score": -2.578839659618401 + }, + { + "epoch": 8, + "test_score": -2.55441917386838, + "train_loss": 0.5041662433125951, + "val_score": -2.5950167216973132 + }, + { + "epoch": 9, + "test_score": -2.5487520754638724, + "train_loss": 0.5028662115066975, + "val_score": -2.5891763278800166 + }, + { + "epoch": 10, + "test_score": -2.5801767219226126, + "train_loss": 0.5012965699036916, + "val_score": -2.6218624437506626 + }, + { + "epoch": 11, + "test_score": -2.5697164569101383, + "train_loss": 0.5027571734007414, + "val_score": -2.610265340069635 + }, + { + "epoch": 12, + "test_score": -2.5612925964655955, + "train_loss": 0.5003397878225859, + "val_score": -2.602886882391948 + }, + { + "epoch": 13, + "test_score": -2.598473900627761, + "train_loss": 0.5009741587144835, + "val_score": -2.6391601110805962 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.539276959092014, + "score": -2.539276959092014 + }, + "val": { + "rmse": 2.578839659618401, + "score": -2.578839659618401 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 39.26718091964722, + "variant": "mfb_only" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2.toml new file mode 100644 index 000000000..fac1025ac --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2.toml @@ -0,0 +1,29 @@ +dataset = "cooking-time" +variant = "mfb_only" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2/report.json new file mode 100644 index 000000000..d5647b82d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2/report.json @@ -0,0 +1,142 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 7, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "mfb_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.5842396674188115, + "train_loss": 0.5787746052484255, + "val_score": -2.6294725553506053 + }, + { + "epoch": 1, + "test_score": -2.5609527196426085, + "train_loss": 0.5174002875615885, + "val_score": -2.6062741614669447 + }, + { + "epoch": 2, + "test_score": -2.5834044067286164, + "train_loss": 0.511275235060099, + "val_score": -2.6247500346015094 + }, + { + "epoch": 3, + "test_score": -2.576294682766308, + "train_loss": 0.5079844250335349, + "val_score": -2.6197295184692186 + }, + { + "epoch": 4, + "test_score": -2.573368285127898, + "train_loss": 0.5065022978159759, + "val_score": -2.614125205300859 + }, + { + "epoch": 5, + "test_score": -2.5507163817962533, + "train_loss": 0.5062107411590783, + "val_score": -2.5932705907251505 + }, + { + "epoch": 6, + "test_score": -2.5672484341318604, + "train_loss": 0.5054079772115828, + "val_score": -2.606842731064844 + }, + { + "epoch": 7, + "test_score": -2.5386993618023146, + "train_loss": 0.505251935204944, + "val_score": -2.577726668594756 + }, + { + "epoch": 8, + "test_score": -2.5609610984138573, + "train_loss": 0.5025441719068063, + "val_score": -2.6023546450549193 + }, + { + "epoch": 9, + "test_score": -2.562959676311487, + "train_loss": 0.5021548641694559, + "val_score": -2.6056102159519456 + }, + { + "epoch": 10, + "test_score": -2.5657840247360215, + "train_loss": 0.5002038159348943, + "val_score": -2.607008357600551 + }, + { + "epoch": 11, + "test_score": -2.574676334075847, + "train_loss": 0.4997164035165632, + "val_score": -2.6152728464660053 + }, + { + "epoch": 12, + "test_score": -2.5638825943321844, + "train_loss": 0.5002949726474177, + "val_score": -2.6052825266388324 + }, + { + "epoch": 13, + "test_score": -2.5865687035005402, + "train_loss": 0.49844017893344433, + "val_score": -2.6283478051631612 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.5386993618023146, + "score": -2.5386993618023146 + }, + "val": { + "rmse": 2.577726668594756, + "score": -2.577726668594756 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 41.57825565338135, + "variant": "mfb_only" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0.toml new file mode 100644 index 000000000..689ec16cd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0.toml @@ -0,0 +1,30 @@ +dataset = "cooking-time" +variant = "rla_only" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0/report.json new file mode 100644 index 000000000..fbf3b0516 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0/report.json @@ -0,0 +1,119 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 3, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "rla_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.6067414842618137, + "train_loss": 0.5621826989693685, + "val_score": -2.648587653030598 + }, + { + "epoch": 1, + "test_score": -2.589142164939684, + "train_loss": 0.49587324476456857, + "val_score": -2.626401890207494 + }, + { + "epoch": 2, + "test_score": -2.5718627706911366, + "train_loss": 0.48924584055806064, + "val_score": -2.606097327135619 + }, + { + "epoch": 3, + "test_score": -2.5699112877582024, + "train_loss": 0.4867811350672095, + "val_score": -2.6052357628095346 + }, + { + "epoch": 4, + "test_score": -2.5745107574860464, + "train_loss": 0.485320704208838, + "val_score": -2.609898040796941 + }, + { + "epoch": 5, + "test_score": -2.587636798430576, + "train_loss": 0.4839465403342032, + "val_score": -2.624125743783909 + }, + { + "epoch": 6, + "test_score": -2.584093986833313, + "train_loss": 0.48281669187116194, + "val_score": -2.619210170473965 + }, + { + "epoch": 7, + "test_score": -2.5796617956053485, + "train_loss": 0.4819454647399284, + "val_score": -2.612544710059628 + }, + { + "epoch": 8, + "test_score": -2.5948246947005935, + "train_loss": 0.4810482220606761, + "val_score": -2.630481083894551 + }, + { + "epoch": 9, + "test_score": -2.590343490417043, + "train_loss": 0.4809136336988157, + "val_score": -2.6232398353437167 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.5699112877582024, + "score": -2.5699112877582024 + }, + "val": { + "rmse": 2.6052357628095346, + "score": -2.6052357628095346 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 28.367936372756958, + "variant": "rla_only" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1.toml new file mode 100644 index 000000000..0f92ae19e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1.toml @@ -0,0 +1,30 @@ +dataset = "cooking-time" +variant = "rla_only" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1/report.json new file mode 100644 index 000000000..f3d4ce614 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1/report.json @@ -0,0 +1,119 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 3, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "rla_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.574028969051126, + "train_loss": 0.5646411073100459, + "val_score": -2.6166735699637482 + }, + { + "epoch": 1, + "test_score": -2.5752987253186155, + "train_loss": 0.49558495642902617, + "val_score": -2.6112346304317455 + }, + { + "epoch": 2, + "test_score": -2.553700761063615, + "train_loss": 0.48945968484019375, + "val_score": -2.592604510592764 + }, + { + "epoch": 3, + "test_score": -2.5449543831564863, + "train_loss": 0.4868325668949265, + "val_score": -2.5806605037686903 + }, + { + "epoch": 4, + "test_score": -2.5839795770694307, + "train_loss": 0.48536315828830273, + "val_score": -2.6187721124047783 + }, + { + "epoch": 5, + "test_score": -2.5676175630837914, + "train_loss": 0.4838317922643713, + "val_score": -2.6051026048423873 + }, + { + "epoch": 6, + "test_score": -2.583028119012807, + "train_loss": 0.4833074563795382, + "val_score": -2.6181113363361583 + }, + { + "epoch": 7, + "test_score": -2.575665219499035, + "train_loss": 0.48161201272998844, + "val_score": -2.6116000071743652 + }, + { + "epoch": 8, + "test_score": -2.5636876773187365, + "train_loss": 0.4816002647082011, + "val_score": -2.597791074538032 + }, + { + "epoch": 9, + "test_score": -2.5757077068628393, + "train_loss": 0.4801698541856027, + "val_score": -2.612110097892733 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.5449543831564863, + "score": -2.5449543831564863 + }, + "val": { + "rmse": 2.5806605037686903, + "score": -2.5806605037686903 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 30.289871215820312, + "variant": "rla_only" +} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2.toml new file mode 100644 index 000000000..1187840b4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2.toml @@ -0,0 +1,30 @@ +dataset = "cooking-time" +variant = "rla_only" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/cooking-time" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2/report.json new file mode 100644 index 000000000..e9954ed1f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2/report.json @@ -0,0 +1,107 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 1, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/cooking-time" + }, + "dataset": "cooking-time", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "rla_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2.toml", + "dataset": "cooking-time", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -2.6212606590101575, + "train_loss": 0.5664722629495569, + "val_score": -2.6634027553513446 + }, + { + "epoch": 1, + "test_score": -2.5677737420490234, + "train_loss": 0.49630044951095237, + "val_score": -2.6059867195323827 + }, + { + "epoch": 2, + "test_score": -2.5929284690585463, + "train_loss": 0.48952298738934974, + "val_score": -2.6290505356892178 + }, + { + "epoch": 3, + "test_score": -2.5822994768076653, + "train_loss": 0.48763485615317886, + "val_score": -2.6185297470515487 + }, + { + "epoch": 4, + "test_score": -2.5928140813080858, + "train_loss": 0.4848814955702773, + "val_score": -2.628734384618246 + }, + { + "epoch": 5, + "test_score": -2.5781435879123955, + "train_loss": 0.48458673368703137, + "val_score": -2.6120996013245725 + }, + { + "epoch": 6, + "test_score": -2.582673655134024, + "train_loss": 0.48272706715910285, + "val_score": -2.6179256480057504 + }, + { + "epoch": 7, + "test_score": -2.579921674352669, + "train_loss": 0.4819437984947686, + "val_score": -2.6134262158794432 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 2.5677737420490234, + "score": -2.5677737420490234 + }, + "val": { + "rmse": 2.6059867195323827, + "score": -2.6059867195323827 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 25.377673625946045, + "variant": "rla_only" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..2ea3ddbd7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0.toml @@ -0,0 +1,35 @@ +dataset = "delivery-eta" +variant = "all_four_combined" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..70ac8fe4f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0/report.json @@ -0,0 +1,106 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_four_combined" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.8656861082768745, + "train_loss": 0.6351582882178091, + "val_score": -1.7428132032666934 + }, + { + "epoch": 1, + "test_score": -1.9173332451200045, + "train_loss": 0.5862035707835733, + "val_score": -1.8012485518439756 + }, + { + "epoch": 2, + "test_score": -1.9211029735094005, + "train_loss": 0.5774363997208811, + "val_score": -1.8070825974375915 + }, + { + "epoch": 3, + "test_score": -1.9158700932442214, + "train_loss": 0.5784730580601379, + "val_score": -1.806458566101268 + }, + { + "epoch": 4, + "test_score": -1.9295777818631115, + "train_loss": 0.5680552694919335, + "val_score": -1.8254905694547126 + }, + { + "epoch": 5, + "test_score": -1.900368983428771, + "train_loss": 0.5624197795443291, + "val_score": -1.8009521006135125 + }, + { + "epoch": 6, + "test_score": -1.9369851935849267, + "train_loss": 0.55270509258674, + "val_score": -1.8281875338822213 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.8656861082768745, + "score": -1.8656861082768745 + }, + "val": { + "rmse": 1.7428132032666934, + "score": -1.7428132032666934 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 37.28859186172485, + "variant": "all_four_combined" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1.toml new file mode 100644 index 000000000..5d9c87b30 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1.toml @@ -0,0 +1,35 @@ +dataset = "delivery-eta" +variant = "all_four_combined" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1/report.json new file mode 100644 index 000000000..42a14dd3f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1/report.json @@ -0,0 +1,112 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 1, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_four_combined" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.9131975360467681, + "train_loss": 0.6403110505890672, + "val_score": -1.8000882153617102 + }, + { + "epoch": 1, + "test_score": -1.8804933819744376, + "train_loss": 0.595119994922276, + "val_score": -1.770191195878174 + }, + { + "epoch": 2, + "test_score": -1.908992732814027, + "train_loss": 0.5801966007608567, + "val_score": -1.7980903164432638 + }, + { + "epoch": 3, + "test_score": -1.922019580892163, + "train_loss": 0.5665582211783332, + "val_score": -1.8065416463596233 + }, + { + "epoch": 4, + "test_score": -1.9283695430647496, + "train_loss": 0.5643883639008459, + "val_score": -1.8264561396192909 + }, + { + "epoch": 5, + "test_score": -1.8923228813143904, + "train_loss": 0.564144954629188, + "val_score": -1.7846969581263876 + }, + { + "epoch": 6, + "test_score": -1.9317067195743272, + "train_loss": 0.5586930552538294, + "val_score": -1.8305701181735838 + }, + { + "epoch": 7, + "test_score": -1.9308092200702978, + "train_loss": 0.5511535132018319, + "val_score": -1.8270347800449493 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.8804933819744376, + "score": -1.8804933819744376 + }, + "val": { + "rmse": 1.770191195878174, + "score": -1.770191195878174 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 43.716530084609985, + "variant": "all_four_combined" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2.toml new file mode 100644 index 000000000..df6430d22 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2.toml @@ -0,0 +1,35 @@ +dataset = "delivery-eta" +variant = "all_four_combined" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2/report.json new file mode 100644 index 000000000..ab7cd1f8b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2/report.json @@ -0,0 +1,106 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_four_combined" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.8965344093393364, + "train_loss": 0.6273651157852507, + "val_score": -1.776750738124176 + }, + { + "epoch": 1, + "test_score": -1.9010519854818586, + "train_loss": 0.5775207037473247, + "val_score": -1.7887463019524075 + }, + { + "epoch": 2, + "test_score": -1.9282420687013453, + "train_loss": 0.578986707830081, + "val_score": -1.8215971903394392 + }, + { + "epoch": 3, + "test_score": -1.9296743415661881, + "train_loss": 0.5668568354453484, + "val_score": -1.8227773849319975 + }, + { + "epoch": 4, + "test_score": -1.9421842002687175, + "train_loss": 0.5612639053894656, + "val_score": -1.8275582502816723 + }, + { + "epoch": 5, + "test_score": -1.8922988795820175, + "train_loss": 0.564061529444952, + "val_score": -1.7890262516797284 + }, + { + "epoch": 6, + "test_score": -1.886318187959984, + "train_loss": 0.5608200274244712, + "val_score": -1.782879134595343 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.8965344093393364, + "score": -1.8965344093393364 + }, + "val": { + "rmse": 1.776750738124176, + "score": -1.776750738124176 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 38.261871099472046, + "variant": "all_four_combined" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..1620a13b0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0.toml @@ -0,0 +1,34 @@ +dataset = "delivery-eta" +variant = "all_minus_cf_fisd" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..04525af19 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0/report.json @@ -0,0 +1,105 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_minus_cf_fisd" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.891667480580607, + "train_loss": 0.6645825640128477, + "val_score": -1.7783494723849997 + }, + { + "epoch": 1, + "test_score": -1.9410785728560749, + "train_loss": 0.5880636803425141, + "val_score": -1.8319506271614023 + }, + { + "epoch": 2, + "test_score": -1.8896939215496575, + "train_loss": 0.5807777503981207, + "val_score": -1.7790821335280576 + }, + { + "epoch": 3, + "test_score": -1.9174438504150855, + "train_loss": 0.585850615571015, + "val_score": -1.800542852877492 + }, + { + "epoch": 4, + "test_score": -1.885224566148601, + "train_loss": 0.5778306814005775, + "val_score": -1.7802418315214172 + }, + { + "epoch": 5, + "test_score": -1.9026999012896584, + "train_loss": 0.5774297648972838, + "val_score": -1.8049066583903193 + }, + { + "epoch": 6, + "test_score": -1.9138769565881466, + "train_loss": 0.5662677301107532, + "val_score": -1.8001846351236526 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.891667480580607, + "score": -1.891667480580607 + }, + "val": { + "rmse": 1.7783494723849997, + "score": -1.7783494723849997 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 38.053216218948364, + "variant": "all_minus_cf_fisd" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1.toml new file mode 100644 index 000000000..a96c2c016 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1.toml @@ -0,0 +1,34 @@ +dataset = "delivery-eta" +variant = "all_minus_cf_fisd" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1/report.json new file mode 100644 index 000000000..ceb35433c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1/report.json @@ -0,0 +1,111 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 1, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_minus_cf_fisd" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.9505100292387032, + "train_loss": 0.6705632031398968, + "val_score": -1.8468169872899822 + }, + { + "epoch": 1, + "test_score": -1.8636960381640717, + "train_loss": 0.5966045582381478, + "val_score": -1.759412249480788 + }, + { + "epoch": 2, + "test_score": -1.906402769377118, + "train_loss": 0.5835885000924994, + "val_score": -1.7950902949072276 + }, + { + "epoch": 3, + "test_score": -1.8905796644591906, + "train_loss": 0.5713840862260248, + "val_score": -1.7731201291210676 + }, + { + "epoch": 4, + "test_score": -1.9279880221536947, + "train_loss": 0.5733988989878746, + "val_score": -1.8203217338360556 + }, + { + "epoch": 5, + "test_score": -1.9064170889244985, + "train_loss": 0.578271459271438, + "val_score": -1.7922601899602175 + }, + { + "epoch": 6, + "test_score": -1.922968979520343, + "train_loss": 0.5738831989086457, + "val_score": -1.8224447965699262 + }, + { + "epoch": 7, + "test_score": -1.9305441495465439, + "train_loss": 0.5651097101886777, + "val_score": -1.8236467334566928 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.8636960381640717, + "score": -1.8636960381640717 + }, + "val": { + "rmse": 1.759412249480788, + "score": -1.759412249480788 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 43.60566973686218, + "variant": "all_minus_cf_fisd" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2.toml new file mode 100644 index 000000000..43d746891 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2.toml @@ -0,0 +1,34 @@ +dataset = "delivery-eta" +variant = "all_minus_cf_fisd" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2/report.json new file mode 100644 index 000000000..6a9d80a6b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2/report.json @@ -0,0 +1,111 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 1, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_minus_cf_fisd" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.8714515168791568, + "train_loss": 0.6583039582210736, + "val_score": -1.757904253790227 + }, + { + "epoch": 1, + "test_score": -1.8633819488889196, + "train_loss": 0.578686815108696, + "val_score": -1.7545374221006966 + }, + { + "epoch": 2, + "test_score": -1.9262881588730187, + "train_loss": 0.5821841881222969, + "val_score": -1.8210502079259065 + }, + { + "epoch": 3, + "test_score": -1.938032384761293, + "train_loss": 0.5721039728526651, + "val_score": -1.826789824842461 + }, + { + "epoch": 4, + "test_score": -1.9325213916971438, + "train_loss": 0.5703568275827561, + "val_score": -1.8151809333173128 + }, + { + "epoch": 5, + "test_score": -1.9152714862063926, + "train_loss": 0.5760056446068478, + "val_score": -1.8096439616269226 + }, + { + "epoch": 6, + "test_score": -1.890280637038989, + "train_loss": 0.5767811636855132, + "val_score": -1.7839441504139821 + }, + { + "epoch": 7, + "test_score": -1.9192103079194238, + "train_loss": 0.5665370217204964, + "val_score": -1.8114721080160514 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.8633819488889196, + "score": -1.8633819488889196 + }, + "val": { + "rmse": 1.7545374221006966, + "score": -1.7545374221006966 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 43.14394950866699, + "variant": "all_minus_cf_fisd" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0.toml new file mode 100644 index 000000000..72936a3e7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0.toml @@ -0,0 +1,33 @@ +dataset = "delivery-eta" +variant = "all_minus_esam" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0/report.json new file mode 100644 index 000000000..6569eeaab --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0/report.json @@ -0,0 +1,104 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_minus_esam" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.8574282929864145, + "train_loss": 0.6346418135357599, + "val_score": -1.734733819358604 + }, + { + "epoch": 1, + "test_score": -1.9188538663007366, + "train_loss": 0.5895762147694609, + "val_score": -1.8017549674986006 + }, + { + "epoch": 2, + "test_score": -1.921810118037487, + "train_loss": 0.5745818023263973, + "val_score": -1.8123889099803876 + }, + { + "epoch": 3, + "test_score": -1.8605371257519345, + "train_loss": 0.576130695586657, + "val_score": -1.7577453600459139 + }, + { + "epoch": 4, + "test_score": -1.944784069676435, + "train_loss": 0.5613377011605423, + "val_score": -1.8333236592933424 + }, + { + "epoch": 5, + "test_score": -1.9003918168474871, + "train_loss": 0.5654894726989913, + "val_score": -1.7906525055535318 + }, + { + "epoch": 6, + "test_score": -1.9127423151629934, + "train_loss": 0.5558785330640138, + "val_score": -1.8136087512193444 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.8574282929864145, + "score": -1.8574282929864145 + }, + "val": { + "rmse": 1.734733819358604, + "score": -1.734733819358604 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 25.650846242904663, + "variant": "all_minus_esam" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1.toml new file mode 100644 index 000000000..475232b10 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1.toml @@ -0,0 +1,33 @@ +dataset = "delivery-eta" +variant = "all_minus_esam" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1/report.json new file mode 100644 index 000000000..26eb815f7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1/report.json @@ -0,0 +1,122 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 3, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_minus_esam" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.9402282948223006, + "train_loss": 0.6321390519176957, + "val_score": -1.8233988387610958 + }, + { + "epoch": 1, + "test_score": -1.913040261845467, + "train_loss": 0.5818619223406715, + "val_score": -1.8055431104703896 + }, + { + "epoch": 2, + "test_score": -1.8981380775963306, + "train_loss": 0.5805560267754715, + "val_score": -1.78609603752208 + }, + { + "epoch": 3, + "test_score": -1.8896128569809156, + "train_loss": 0.5727628110534083, + "val_score": -1.7733117953883153 + }, + { + "epoch": 4, + "test_score": -1.914617652571641, + "train_loss": 0.561080602398754, + "val_score": -1.8137358036971076 + }, + { + "epoch": 5, + "test_score": -1.917097900087658, + "train_loss": 0.5638521348472929, + "val_score": -1.8097799213716703 + }, + { + "epoch": 6, + "test_score": -1.899768378545227, + "train_loss": 0.5505518221507107, + "val_score": -1.7935067493704147 + }, + { + "epoch": 7, + "test_score": -1.9753858961878896, + "train_loss": 0.5567100800736977, + "val_score": -1.8744500943284486 + }, + { + "epoch": 8, + "test_score": -1.9289701417992142, + "train_loss": 0.5494376259998683, + "val_score": -1.8191711383903835 + }, + { + "epoch": 9, + "test_score": -1.9488545232297387, + "train_loss": 0.5475168900333182, + "val_score": -1.8421066201714127 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.8896128569809156, + "score": -1.8896128569809156 + }, + "val": { + "rmse": 1.7733117953883153, + "score": -1.7733117953883153 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 37.29048800468445, + "variant": "all_minus_esam" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2.toml new file mode 100644 index 000000000..eb4d035c6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2.toml @@ -0,0 +1,33 @@ +dataset = "delivery-eta" +variant = "all_minus_esam" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2/report.json new file mode 100644 index 000000000..80251e8da --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2/report.json @@ -0,0 +1,104 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_minus_esam" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.889693353794343, + "train_loss": 0.6369702032882801, + "val_score": -1.7678478056315658 + }, + { + "epoch": 1, + "test_score": -1.928198915916465, + "train_loss": 0.5902615276566388, + "val_score": -1.8122465022165561 + }, + { + "epoch": 2, + "test_score": -1.9130482380274412, + "train_loss": 0.5736563127406322, + "val_score": -1.7966875724897298 + }, + { + "epoch": 3, + "test_score": -1.9379275680800756, + "train_loss": 0.5651734101511267, + "val_score": -1.8271425655888411 + }, + { + "epoch": 4, + "test_score": -1.9250888779174509, + "train_loss": 0.570913371378488, + "val_score": -1.8123990392518639 + }, + { + "epoch": 5, + "test_score": -1.913662366498882, + "train_loss": 0.5671725908335108, + "val_score": -1.8027153303436068 + }, + { + "epoch": 6, + "test_score": -1.8924741293223586, + "train_loss": 0.5528815042798536, + "val_score": -1.7809590553790522 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.889693353794343, + "score": -1.889693353794343 + }, + "val": { + "rmse": 1.7678478056315658, + "score": -1.7678478056315658 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 26.155076503753662, + "variant": "all_minus_esam" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0.toml new file mode 100644 index 000000000..0b7668306 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0.toml @@ -0,0 +1,33 @@ +dataset = "delivery-eta" +variant = "all_minus_mfb" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0/report.json new file mode 100644 index 000000000..8b15b2ac3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0/report.json @@ -0,0 +1,128 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 4, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_minus_mfb" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.888090720801521, + "train_loss": 0.5960141698809436, + "val_score": -1.7678867133944087 + }, + { + "epoch": 1, + "test_score": -1.9079790872111537, + "train_loss": 0.545285747434101, + "val_score": -1.8052117713268874 + }, + { + "epoch": 2, + "test_score": -1.8922816183054478, + "train_loss": 0.5397576872449722, + "val_score": -1.7867413251257125 + }, + { + "epoch": 3, + "test_score": -1.9028843419042973, + "train_loss": 0.536361330399548, + "val_score": -1.7928649606355145 + }, + { + "epoch": 4, + "test_score": -1.8611355949170485, + "train_loss": 0.5343982783982354, + "val_score": -1.7538454856626753 + }, + { + "epoch": 5, + "test_score": -1.8868397423153196, + "train_loss": 0.5325560691582896, + "val_score": -1.7828675671965248 + }, + { + "epoch": 6, + "test_score": -1.9080221349963706, + "train_loss": 0.5307190265968769, + "val_score": -1.8020101396400294 + }, + { + "epoch": 7, + "test_score": -1.9162634826305283, + "train_loss": 0.5294287024188216, + "val_score": -1.8024161441254358 + }, + { + "epoch": 8, + "test_score": -1.9172990488246484, + "train_loss": 0.5285489454756688, + "val_score": -1.8042130940772272 + }, + { + "epoch": 9, + "test_score": -1.8960975700374516, + "train_loss": 0.5278546320695947, + "val_score": -1.792183697936757 + }, + { + "epoch": 10, + "test_score": -1.8802992639365812, + "train_loss": 0.5262137990798393, + "val_score": -1.7697711975578432 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.8611355949170485, + "score": -1.8611355949170485 + }, + "val": { + "rmse": 1.7538454856626753, + "score": -1.7538454856626753 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 58.886919260025024, + "variant": "all_minus_mfb" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1.toml new file mode 100644 index 000000000..bb9ee9d12 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1.toml @@ -0,0 +1,33 @@ +dataset = "delivery-eta" +variant = "all_minus_mfb" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1/report.json new file mode 100644 index 000000000..fdd5787b4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1/report.json @@ -0,0 +1,116 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 2, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_minus_mfb" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.9291432960802894, + "train_loss": 0.5934300527085353, + "val_score": -1.8205802622313019 + }, + { + "epoch": 1, + "test_score": -1.8930283710632847, + "train_loss": 0.544890384604461, + "val_score": -1.7886505318955528 + }, + { + "epoch": 2, + "test_score": -1.8741570167185646, + "train_loss": 0.5395748837150797, + "val_score": -1.7677035629696638 + }, + { + "epoch": 3, + "test_score": -1.9244894837002682, + "train_loss": 0.5360066024926458, + "val_score": -1.815649518948847 + }, + { + "epoch": 4, + "test_score": -1.9197741555463843, + "train_loss": 0.5336031124104549, + "val_score": -1.8135653686143167 + }, + { + "epoch": 5, + "test_score": -1.927351367627674, + "train_loss": 0.5316786544166342, + "val_score": -1.8143061484933616 + }, + { + "epoch": 6, + "test_score": -1.919555504924725, + "train_loss": 0.5302330420835175, + "val_score": -1.8160835225522036 + }, + { + "epoch": 7, + "test_score": -1.9236640356734735, + "train_loss": 0.5287505892941552, + "val_score": -1.8162988774548237 + }, + { + "epoch": 8, + "test_score": -1.945460608792154, + "train_loss": 0.5278126967214319, + "val_score": -1.8351122360830159 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.8741570167185646, + "score": -1.8741570167185646 + }, + "val": { + "rmse": 1.7677035629696638, + "score": -1.7677035629696638 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 47.90096688270569, + "variant": "all_minus_mfb" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2.toml new file mode 100644 index 000000000..a02615d6f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2.toml @@ -0,0 +1,33 @@ +dataset = "delivery-eta" +variant = "all_minus_mfb" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2/report.json new file mode 100644 index 000000000..428426e0a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2/report.json @@ -0,0 +1,104 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_minus_mfb" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.870944405411248, + "train_loss": 0.5949648753569944, + "val_score": -1.7558893831296303 + }, + { + "epoch": 1, + "test_score": -1.898728148263703, + "train_loss": 0.5453242161413179, + "val_score": -1.7877388934576073 + }, + { + "epoch": 2, + "test_score": -1.9169362818397024, + "train_loss": 0.540534051230354, + "val_score": -1.8111667333277426 + }, + { + "epoch": 3, + "test_score": -1.9006590854507406, + "train_loss": 0.5365165554694016, + "val_score": -1.7956854822081687 + }, + { + "epoch": 4, + "test_score": -1.9391828427407138, + "train_loss": 0.5344064231336552, + "val_score": -1.8293733165799688 + }, + { + "epoch": 5, + "test_score": -1.9016114378299895, + "train_loss": 0.532535148580579, + "val_score": -1.793748806411399 + }, + { + "epoch": 6, + "test_score": -1.9103862101360687, + "train_loss": 0.5309882124845129, + "val_score": -1.7998801270941822 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.870944405411248, + "score": -1.870944405411248 + }, + "val": { + "rmse": 1.7558893831296303, + "score": -1.7558893831296303 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 38.0086407661438, + "variant": "all_minus_mfb" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0.toml new file mode 100644 index 000000000..a8e54a528 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0.toml @@ -0,0 +1,32 @@ +dataset = "delivery-eta" +variant = "all_minus_rla" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0/report.json new file mode 100644 index 000000000..438690292 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0/report.json @@ -0,0 +1,103 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_minus_rla" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.860688971411864, + "train_loss": 0.6334640344564062, + "val_score": -1.7427255800271264 + }, + { + "epoch": 1, + "test_score": -1.9110673163198892, + "train_loss": 0.587792286472599, + "val_score": -1.7940440558272999 + }, + { + "epoch": 2, + "test_score": -1.905961249322842, + "train_loss": 0.5795968405521699, + "val_score": -1.7906092325152145 + }, + { + "epoch": 3, + "test_score": -1.9461060403001613, + "train_loss": 0.5841769684840293, + "val_score": -1.8283881625656664 + }, + { + "epoch": 4, + "test_score": -1.9359065240409257, + "train_loss": 0.5748188164982483, + "val_score": -1.8243624448850637 + }, + { + "epoch": 5, + "test_score": -1.9230347522366662, + "train_loss": 0.5720745399050469, + "val_score": -1.8228242759906252 + }, + { + "epoch": 6, + "test_score": -1.9510359922028844, + "train_loss": 0.5629213082529333, + "val_score": -1.8318482655318722 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.860688971411864, + "score": -1.860688971411864 + }, + "val": { + "rmse": 1.7427255800271264, + "score": -1.7427255800271264 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 33.61688852310181, + "variant": "all_minus_rla" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1.toml new file mode 100644 index 000000000..b36c0e027 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1.toml @@ -0,0 +1,32 @@ +dataset = "delivery-eta" +variant = "all_minus_rla" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1/report.json new file mode 100644 index 000000000..9f58cc0b6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1/report.json @@ -0,0 +1,109 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 1, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_minus_rla" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.8917534983307565, + "train_loss": 0.6392366142168532, + "val_score": -1.7708043114771035 + }, + { + "epoch": 1, + "test_score": -1.8698356715330888, + "train_loss": 0.5946752181018355, + "val_score": -1.7504887579540844 + }, + { + "epoch": 2, + "test_score": -1.9423982169881553, + "train_loss": 0.5839949937632484, + "val_score": -1.8282106167560017 + }, + { + "epoch": 3, + "test_score": -1.9397289621633413, + "train_loss": 0.5700914755354832, + "val_score": -1.8241841151607034 + }, + { + "epoch": 4, + "test_score": -1.9145957982346236, + "train_loss": 0.5711912164722917, + "val_score": -1.8062533896655315 + }, + { + "epoch": 5, + "test_score": -1.8782919912631686, + "train_loss": 0.5749758856139914, + "val_score": -1.7671069787094078 + }, + { + "epoch": 6, + "test_score": -1.9406991204252366, + "train_loss": 0.5706792591262038, + "val_score": -1.8289964343245315 + }, + { + "epoch": 7, + "test_score": -1.92082874383054, + "train_loss": 0.5622948334164863, + "val_score": -1.805523699291935 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.8698356715330888, + "score": -1.8698356715330888 + }, + "val": { + "rmse": 1.7504887579540844, + "score": -1.7504887579540844 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 37.817176818847656, + "variant": "all_minus_rla" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2.toml new file mode 100644 index 000000000..16b347221 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2.toml @@ -0,0 +1,32 @@ +dataset = "delivery-eta" +variant = "all_minus_rla" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2/report.json new file mode 100644 index 000000000..8bf5ea3bd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2/report.json @@ -0,0 +1,103 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_minus_rla" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.9066122992751606, + "train_loss": 0.6281222657565653, + "val_score": -1.7852811206241392 + }, + { + "epoch": 1, + "test_score": -1.9025621230099772, + "train_loss": 0.5781265331880889, + "val_score": -1.7918848156269322 + }, + { + "epoch": 2, + "test_score": -1.932680473043221, + "train_loss": 0.5821000093961284, + "val_score": -1.8283426530172184 + }, + { + "epoch": 3, + "test_score": -1.923708591523656, + "train_loss": 0.5710510563676374, + "val_score": -1.80417952881678 + }, + { + "epoch": 4, + "test_score": -1.9435592165228985, + "train_loss": 0.5663256745268829, + "val_score": -1.8293261372329226 + }, + { + "epoch": 5, + "test_score": -1.905806505486743, + "train_loss": 0.5734849132760598, + "val_score": -1.7990136729087136 + }, + { + "epoch": 6, + "test_score": -1.8987611722334956, + "train_loss": 0.5716026260034881, + "val_score": -1.7859538026668018 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.9066122992751606, + "score": -1.9066122992751606 + }, + "val": { + "rmse": 1.7852811206241392, + "score": -1.7852811206241392 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 32.113162994384766, + "variant": "all_minus_rla" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..4dc608f3b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0.toml @@ -0,0 +1,27 @@ +dataset = "delivery-eta" +variant = "baseline_plr" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..327d30f92 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0/report.json @@ -0,0 +1,98 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "baseline_plr" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.8396023796480883, + "train_loss": 0.6278780659619909, + "val_score": -1.7395649830960092 + }, + { + "epoch": 1, + "test_score": -1.889375825680506, + "train_loss": 0.5447326698442445, + "val_score": -1.7913296922311428 + }, + { + "epoch": 2, + "test_score": -1.9264100694336173, + "train_loss": 0.5396799610479035, + "val_score": -1.8183812059183937 + }, + { + "epoch": 3, + "test_score": -1.9165897423055016, + "train_loss": 0.53633236558768, + "val_score": -1.8094179981768903 + }, + { + "epoch": 4, + "test_score": -1.9316801215628367, + "train_loss": 0.5344857908513424, + "val_score": -1.823777596759711 + }, + { + "epoch": 5, + "test_score": -1.9056037057379331, + "train_loss": 0.5330761410024044, + "val_score": -1.797133319365044 + }, + { + "epoch": 6, + "test_score": -1.9031948559097036, + "train_loss": 0.5317075433087175, + "val_score": -1.802448750162817 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.8396023796480883, + "score": -1.8396023796480883 + }, + "val": { + "rmse": 1.7395649830960092, + "score": -1.7395649830960092 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 22.571192026138306, + "variant": "baseline_plr" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1.toml new file mode 100644 index 000000000..4eb0c2a12 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1.toml @@ -0,0 +1,27 @@ +dataset = "delivery-eta" +variant = "baseline_plr" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1/report.json new file mode 100644 index 000000000..eb6514129 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1/report.json @@ -0,0 +1,98 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "baseline_plr" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.8639852605326992, + "train_loss": 0.6245106560905485, + "val_score": -1.7606024408977228 + }, + { + "epoch": 1, + "test_score": -1.8734272400588596, + "train_loss": 0.5430235405908014, + "val_score": -1.7644821113940043 + }, + { + "epoch": 2, + "test_score": -1.946784936771937, + "train_loss": 0.5393460347269573, + "val_score": -1.8353593930658616 + }, + { + "epoch": 3, + "test_score": -1.9135835009303772, + "train_loss": 0.5361843874854763, + "val_score": -1.8047017679022026 + }, + { + "epoch": 4, + "test_score": -1.8845223541593812, + "train_loss": 0.5334207896333542, + "val_score": -1.7767816010897883 + }, + { + "epoch": 5, + "test_score": -1.9256172032927374, + "train_loss": 0.5318533973972293, + "val_score": -1.8147362692408484 + }, + { + "epoch": 6, + "test_score": -1.889762240193959, + "train_loss": 0.530044365755833, + "val_score": -1.78181903517071 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.8639852605326992, + "score": -1.8639852605326992 + }, + "val": { + "rmse": 1.7606024408977228, + "score": -1.7606024408977228 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 23.594813346862793, + "variant": "baseline_plr" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2.toml new file mode 100644 index 000000000..bc63f8e11 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2.toml @@ -0,0 +1,27 @@ +dataset = "delivery-eta" +variant = "baseline_plr" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2/report.json new file mode 100644 index 000000000..5ebb516e5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2/report.json @@ -0,0 +1,122 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 4, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "baseline_plr" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.8721481888104075, + "train_loss": 0.6234128196744153, + "val_score": -1.7699674022496135 + }, + { + "epoch": 1, + "test_score": -1.873120446694421, + "train_loss": 0.5432704556597411, + "val_score": -1.773794062727797 + }, + { + "epoch": 2, + "test_score": -1.9151929358757613, + "train_loss": 0.5391251908166566, + "val_score": -1.815582285393361 + }, + { + "epoch": 3, + "test_score": -1.9287470328900895, + "train_loss": 0.5359159618833639, + "val_score": -1.8178189598311036 + }, + { + "epoch": 4, + "test_score": -1.8590972436481992, + "train_loss": 0.5342420942156855, + "val_score": -1.762300314988203 + }, + { + "epoch": 5, + "test_score": -1.8938041628907152, + "train_loss": 0.5325591675121418, + "val_score": -1.7882742665405706 + }, + { + "epoch": 6, + "test_score": -1.9157789980547906, + "train_loss": 0.5309514068338993, + "val_score": -1.8092274554615024 + }, + { + "epoch": 7, + "test_score": -1.9134677507899076, + "train_loss": 0.5297948411346352, + "val_score": -1.8133076151024474 + }, + { + "epoch": 8, + "test_score": -1.944729330749166, + "train_loss": 0.5287812985207913, + "val_score": -1.8367321486087818 + }, + { + "epoch": 9, + "test_score": -1.8971487309694048, + "train_loss": 0.5279093323832881, + "val_score": -1.7899461587874737 + }, + { + "epoch": 10, + "test_score": -1.909957469746283, + "train_loss": 0.5260672956487559, + "val_score": -1.8055805457252359 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.8590972436481992, + "score": -1.8590972436481992 + }, + "val": { + "rmse": 1.762300314988203, + "score": -1.762300314988203 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 36.959254026412964, + "variant": "baseline_plr" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..e3acfbe87 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0.toml @@ -0,0 +1,28 @@ +dataset = "delivery-eta" +variant = "cf_fisd_only" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..61b118e50 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,99 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "cf_fisd_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.847462746808535, + "train_loss": 0.5944465786001109, + "val_score": -1.7329661515326225 + }, + { + "epoch": 1, + "test_score": -1.8863803093999616, + "train_loss": 0.5455469045325787, + "val_score": -1.782074921309916 + }, + { + "epoch": 2, + "test_score": -1.9238514235581599, + "train_loss": 0.5399144372365771, + "val_score": -1.8125378177249112 + }, + { + "epoch": 3, + "test_score": -1.9124455063978703, + "train_loss": 0.53612889512612, + "val_score": -1.7996051779172895 + }, + { + "epoch": 4, + "test_score": -1.9290277998222791, + "train_loss": 0.5339680866603433, + "val_score": -1.8197581220598342 + }, + { + "epoch": 5, + "test_score": -1.9045877557287432, + "train_loss": 0.5323602639845688, + "val_score": -1.7948221172717103 + }, + { + "epoch": 6, + "test_score": -1.9236647173423764, + "train_loss": 0.5312561038201743, + "val_score": -1.8180548948722761 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.847462746808535, + "score": -1.847462746808535 + }, + "val": { + "rmse": 1.7329661515326225, + "score": -1.7329661515326225 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 21.50657367706299, + "variant": "cf_fisd_only" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1.toml new file mode 100644 index 000000000..763e4dbdd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1.toml @@ -0,0 +1,28 @@ +dataset = "delivery-eta" +variant = "cf_fisd_only" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1/report.json new file mode 100644 index 000000000..7bfb46dde --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1/report.json @@ -0,0 +1,105 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 1, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "cf_fisd_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.8773772110895843, + "train_loss": 0.592602454180265, + "val_score": -1.7619405467279132 + }, + { + "epoch": 1, + "test_score": -1.8568220804711988, + "train_loss": 0.5443947880807585, + "val_score": -1.7414478733432992 + }, + { + "epoch": 2, + "test_score": -1.9585844717578214, + "train_loss": 0.5396959011572121, + "val_score": -1.8442746078795882 + }, + { + "epoch": 3, + "test_score": -1.9227801416713162, + "train_loss": 0.5361053360639697, + "val_score": -1.8145431315556086 + }, + { + "epoch": 4, + "test_score": -1.8821978693611068, + "train_loss": 0.5335870689719263, + "val_score": -1.77016809721336 + }, + { + "epoch": 5, + "test_score": -1.9337962034952991, + "train_loss": 0.5318883611773052, + "val_score": -1.8237204677813132 + }, + { + "epoch": 6, + "test_score": -1.9025955816813702, + "train_loss": 0.530138851082238, + "val_score": -1.7919569963123252 + }, + { + "epoch": 7, + "test_score": -1.9315667519170043, + "train_loss": 0.5293886254738717, + "val_score": -1.8192720509490548 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.8568220804711988, + "score": -1.8568220804711988 + }, + "val": { + "rmse": 1.7414478733432992, + "score": -1.7414478733432992 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 27.340318202972412, + "variant": "cf_fisd_only" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2.toml new file mode 100644 index 000000000..0efb09181 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2.toml @@ -0,0 +1,28 @@ +dataset = "delivery-eta" +variant = "cf_fisd_only" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2/report.json new file mode 100644 index 000000000..a5b7220ef --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2/report.json @@ -0,0 +1,123 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 4, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "cf_fisd_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.9230385956246019, + "train_loss": 0.5934791673709007, + "val_score": -1.8042379372536939 + }, + { + "epoch": 1, + "test_score": -1.8948157028119332, + "train_loss": 0.5450501676893582, + "val_score": -1.7880606699001504 + }, + { + "epoch": 2, + "test_score": -1.9230178908310813, + "train_loss": 0.5397137101114231, + "val_score": -1.818952191312291 + }, + { + "epoch": 3, + "test_score": -1.896522906586717, + "train_loss": 0.536448588118936, + "val_score": -1.7856421946551229 + }, + { + "epoch": 4, + "test_score": -1.8737455622217765, + "train_loss": 0.5346145312281421, + "val_score": -1.774137585629917 + }, + { + "epoch": 5, + "test_score": -1.9030765946970747, + "train_loss": 0.5325972875104333, + "val_score": -1.7950323857438173 + }, + { + "epoch": 6, + "test_score": -1.9139981005987072, + "train_loss": 0.5305591375723372, + "val_score": -1.8049593634006922 + }, + { + "epoch": 7, + "test_score": -1.907357628885035, + "train_loss": 0.5295915420908127, + "val_score": -1.8045180600373205 + }, + { + "epoch": 8, + "test_score": -1.9360437150070906, + "train_loss": 0.5289334767926348, + "val_score": -1.8271244277885352 + }, + { + "epoch": 9, + "test_score": -1.913430744154855, + "train_loss": 0.5275534989624998, + "val_score": -1.800273236093376 + }, + { + "epoch": 10, + "test_score": -1.9140615035073936, + "train_loss": 0.5261308607393808, + "val_score": -1.8077536293935552 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.8737455622217765, + "score": -1.8737455622217765 + }, + "val": { + "rmse": 1.774137585629917, + "score": -1.774137585629917 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 36.74504613876343, + "variant": "cf_fisd_only" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0.toml new file mode 100644 index 000000000..a7f0b4166 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0.toml @@ -0,0 +1,29 @@ +dataset = "delivery-eta" +variant = "esam_only" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0/report.json new file mode 100644 index 000000000..2e7ff74b5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0/report.json @@ -0,0 +1,100 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "esam_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.8414060273006467, + "train_loss": 0.6287253525135291, + "val_score": -1.7408771462387886 + }, + { + "epoch": 1, + "test_score": -1.8874872826154119, + "train_loss": 0.5451243132570364, + "val_score": -1.7892239426978986 + }, + { + "epoch": 2, + "test_score": -1.9253018229293297, + "train_loss": 0.5400790667881931, + "val_score": -1.817427678197544 + }, + { + "epoch": 3, + "test_score": -1.915089234546431, + "train_loss": 0.536670585618402, + "val_score": -1.8080386139915399 + }, + { + "epoch": 4, + "test_score": -1.9321186025873545, + "train_loss": 0.5349389023154322, + "val_score": -1.823938711755445 + }, + { + "epoch": 5, + "test_score": -1.905402698756221, + "train_loss": 0.5335488456444148, + "val_score": -1.7974111669286685 + }, + { + "epoch": 6, + "test_score": -1.9018297068818415, + "train_loss": 0.5321317349907255, + "val_score": -1.8006889667158776 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.8414060273006467, + "score": -1.8414060273006467 + }, + "val": { + "rmse": 1.7408771462387886, + "score": -1.7408771462387886 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 31.279943466186523, + "variant": "esam_only" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1.toml new file mode 100644 index 000000000..c42ffbc0f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1.toml @@ -0,0 +1,29 @@ +dataset = "delivery-eta" +variant = "esam_only" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1/report.json new file mode 100644 index 000000000..ba56e807a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1/report.json @@ -0,0 +1,100 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "esam_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.867077237628412, + "train_loss": 0.6253672166027292, + "val_score": -1.7633451748518263 + }, + { + "epoch": 1, + "test_score": -1.8728420555294343, + "train_loss": 0.5435903977303609, + "val_score": -1.7639967575865203 + }, + { + "epoch": 2, + "test_score": -1.9459718865915152, + "train_loss": 0.5396962204988855, + "val_score": -1.8349061057367806 + }, + { + "epoch": 3, + "test_score": -1.9133081311992928, + "train_loss": 0.5365906061917326, + "val_score": -1.8041447075027965 + }, + { + "epoch": 4, + "test_score": -1.8830463058482847, + "train_loss": 0.5339189865728364, + "val_score": -1.7755051713340173 + }, + { + "epoch": 5, + "test_score": -1.925044725567323, + "train_loss": 0.5321700340204866, + "val_score": -1.814003090949406 + }, + { + "epoch": 6, + "test_score": -1.8887670546240742, + "train_loss": 0.5302761634335901, + "val_score": -1.7808658788001235 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.867077237628412, + "score": -1.867077237628412 + }, + "val": { + "rmse": 1.7633451748518263, + "score": -1.7633451748518263 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 33.070332765579224, + "variant": "esam_only" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2.toml new file mode 100644 index 000000000..f5ee21155 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2.toml @@ -0,0 +1,29 @@ +dataset = "delivery-eta" +variant = "esam_only" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2/report.json new file mode 100644 index 000000000..7dd64afb4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2/report.json @@ -0,0 +1,124 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 4, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "esam_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.8725806203887048, + "train_loss": 0.6242489044683693, + "val_score": -1.7706746497769004 + }, + { + "epoch": 1, + "test_score": -1.8729241005732664, + "train_loss": 0.5437626016401026, + "val_score": -1.7736281237893647 + }, + { + "epoch": 2, + "test_score": -1.9148552945994042, + "train_loss": 0.5395388853375929, + "val_score": -1.815533434451519 + }, + { + "epoch": 3, + "test_score": -1.926554681246965, + "train_loss": 0.5361864662518466, + "val_score": -1.815812865105869 + }, + { + "epoch": 4, + "test_score": -1.8608628418981936, + "train_loss": 0.5344241089629431, + "val_score": -1.7636637639971298 + }, + { + "epoch": 5, + "test_score": -1.8918575336215864, + "train_loss": 0.5327327956248374, + "val_score": -1.7864739627729052 + }, + { + "epoch": 6, + "test_score": -1.9155521746136261, + "train_loss": 0.5312667256289155, + "val_score": -1.8085834016771645 + }, + { + "epoch": 7, + "test_score": -1.9122361792814149, + "train_loss": 0.529884976844718, + "val_score": -1.812130528643339 + }, + { + "epoch": 8, + "test_score": -1.9455652034502635, + "train_loss": 0.528975662958883, + "val_score": -1.8374890203861554 + }, + { + "epoch": 9, + "test_score": -1.8964897807836834, + "train_loss": 0.5281089191889241, + "val_score": -1.7885813503347106 + }, + { + "epoch": 10, + "test_score": -1.9095956802829104, + "train_loss": 0.5263300951379929, + "val_score": -1.8048084433187321 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.8608628418981936, + "score": -1.8608628418981936 + }, + "val": { + "rmse": 1.7636637639971298, + "score": -1.7636637639971298 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 53.07820463180542, + "variant": "esam_only" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0.toml new file mode 100644 index 000000000..a05013205 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0.toml @@ -0,0 +1,29 @@ +dataset = "delivery-eta" +variant = "mfb_only" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..41799d2f2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0/report.json @@ -0,0 +1,100 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "mfb_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.882313199140896, + "train_loss": 0.6657763436762956, + "val_score": -1.7760818193507641 + }, + { + "epoch": 1, + "test_score": -1.9108278307008544, + "train_loss": 0.5919238420298499, + "val_score": -1.7950738919278724 + }, + { + "epoch": 2, + "test_score": -1.8997331758164053, + "train_loss": 0.5805986815125402, + "val_score": -1.7929712765079557 + }, + { + "epoch": 3, + "test_score": -1.9036324461863272, + "train_loss": 0.5849480929165861, + "val_score": -1.7954276848902972 + }, + { + "epoch": 4, + "test_score": -1.9270621908591754, + "train_loss": 0.5751130467783796, + "val_score": -1.8031489431817904 + }, + { + "epoch": 5, + "test_score": -1.9185303528159003, + "train_loss": 0.5802982014461155, + "val_score": -1.8075533484134956 + }, + { + "epoch": 6, + "test_score": -1.904226260709987, + "train_loss": 0.5747161829558602, + "val_score": -1.796530184625187 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.882313199140896, + "score": -1.882313199140896 + }, + "val": { + "rmse": 1.7760818193507641, + "score": -1.7760818193507641 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 22.597419023513794, + "variant": "mfb_only" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1.toml new file mode 100644 index 000000000..4c6383fb7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1.toml @@ -0,0 +1,29 @@ +dataset = "delivery-eta" +variant = "mfb_only" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1/report.json new file mode 100644 index 000000000..0b08e793a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1/report.json @@ -0,0 +1,124 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 4, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "mfb_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.9317965700093218, + "train_loss": 0.6622442111481716, + "val_score": -1.8175719099760428 + }, + { + "epoch": 1, + "test_score": -1.9051803339309807, + "train_loss": 0.5844372032332594, + "val_score": -1.7967930650689325 + }, + { + "epoch": 2, + "test_score": -1.9433397455038781, + "train_loss": 0.5849274127152715, + "val_score": -1.8242998452305552 + }, + { + "epoch": 3, + "test_score": -1.9411426265421559, + "train_loss": 0.5797237166523064, + "val_score": -1.8226949139094244 + }, + { + "epoch": 4, + "test_score": -1.8999776362056646, + "train_loss": 0.573150089622414, + "val_score": -1.7831396152179502 + }, + { + "epoch": 5, + "test_score": -1.9150731746547511, + "train_loss": 0.5799409951606807, + "val_score": -1.7980624048862248 + }, + { + "epoch": 6, + "test_score": -1.9013377198618562, + "train_loss": 0.5672564837184265, + "val_score": -1.783942212533094 + }, + { + "epoch": 7, + "test_score": -1.9636375410585374, + "train_loss": 0.579314707839576, + "val_score": -1.8430848942503817 + }, + { + "epoch": 8, + "test_score": -1.930902137414017, + "train_loss": 0.5656810683490586, + "val_score": -1.8118882297797387 + }, + { + "epoch": 9, + "test_score": -1.9378414468459522, + "train_loss": 0.567348591820167, + "val_score": -1.8191206144805452 + }, + { + "epoch": 10, + "test_score": -1.9203693105241764, + "train_loss": 0.5712279420699516, + "val_score": -1.8070823335663142 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.8999776362056646, + "score": -1.8999776362056646 + }, + "val": { + "rmse": 1.7831396152179502, + "score": -1.7831396152179502 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 37.64791131019592, + "variant": "mfb_only" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2.toml new file mode 100644 index 000000000..952dde1cc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2.toml @@ -0,0 +1,29 @@ +dataset = "delivery-eta" +variant = "mfb_only" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2/report.json new file mode 100644 index 000000000..809a7c6bb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2/report.json @@ -0,0 +1,136 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 6, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "mfb_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.9128695137240894, + "train_loss": 0.665501472723745, + "val_score": -1.7964501583575907 + }, + { + "epoch": 1, + "test_score": -1.9440009050249418, + "train_loss": 0.5934047594557713, + "val_score": -1.8344874073011204 + }, + { + "epoch": 2, + "test_score": -1.9288621133395232, + "train_loss": 0.5766440614731643, + "val_score": -1.8188880946637216 + }, + { + "epoch": 3, + "test_score": -1.925762555571344, + "train_loss": 0.5732441340049688, + "val_score": -1.8097932269607562 + }, + { + "epoch": 4, + "test_score": -1.9167776974017394, + "train_loss": 0.586352648961283, + "val_score": -1.8035885986741023 + }, + { + "epoch": 5, + "test_score": -1.926642358861961, + "train_loss": 0.5883107520367977, + "val_score": -1.8093135711309383 + }, + { + "epoch": 6, + "test_score": -1.8822192131865638, + "train_loss": 0.5684957961096381, + "val_score": -1.764251917763006 + }, + { + "epoch": 7, + "test_score": -1.928180183103219, + "train_loss": 0.5747323318989608, + "val_score": -1.8243644705165025 + }, + { + "epoch": 8, + "test_score": -1.9409041494201429, + "train_loss": 0.5614554407822825, + "val_score": -1.8332152617609774 + }, + { + "epoch": 9, + "test_score": -1.9376158525839038, + "train_loss": 0.5655314191414492, + "val_score": -1.820973681432002 + }, + { + "epoch": 10, + "test_score": -1.8920766758614376, + "train_loss": 0.5720564540285263, + "val_score": -1.7866258310993834 + }, + { + "epoch": 11, + "test_score": -1.923358499746884, + "train_loss": 0.5571396168131028, + "val_score": -1.812032902479919 + }, + { + "epoch": 12, + "test_score": -1.9155238586907501, + "train_loss": 0.5610200036181151, + "val_score": -1.8033300141088338 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.8822192131865638, + "score": -1.8822192131865638 + }, + "val": { + "rmse": 1.764251917763006, + "score": -1.764251917763006 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 42.991671562194824, + "variant": "mfb_only" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0.toml new file mode 100644 index 000000000..2daabeb51 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0.toml @@ -0,0 +1,30 @@ +dataset = "delivery-eta" +variant = "rla_only" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0/report.json new file mode 100644 index 000000000..2fe61d66d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0/report.json @@ -0,0 +1,161 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 10, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "rla_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.8991126598508863, + "train_loss": 0.6279589468545287, + "val_score": -1.7893601882007086 + }, + { + "epoch": 1, + "test_score": -1.9124187651691174, + "train_loss": 0.5441900978993325, + "val_score": -1.8123750972462311 + }, + { + "epoch": 2, + "test_score": -1.8848644171404472, + "train_loss": 0.5390364954071324, + "val_score": -1.7817099797140792 + }, + { + "epoch": 3, + "test_score": -1.9074590007130547, + "train_loss": 0.536344505139511, + "val_score": -1.7976066095095695 + }, + { + "epoch": 4, + "test_score": -1.8838248162957778, + "train_loss": 0.5340189500881808, + "val_score": -1.7804151887691884 + }, + { + "epoch": 5, + "test_score": -1.9015802813586458, + "train_loss": 0.532935598688404, + "val_score": -1.796179462659246 + }, + { + "epoch": 6, + "test_score": -1.9064965635839466, + "train_loss": 0.5309413510517482, + "val_score": -1.8006959179170425 + }, + { + "epoch": 7, + "test_score": -1.9036021995153383, + "train_loss": 0.5296575087700447, + "val_score": -1.7933113252903616 + }, + { + "epoch": 8, + "test_score": -1.898886042851269, + "train_loss": 0.5284191111143488, + "val_score": -1.792912966456737 + }, + { + "epoch": 9, + "test_score": -1.8955831537303498, + "train_loss": 0.5278057240656693, + "val_score": -1.793970164966326 + }, + { + "epoch": 10, + "test_score": -1.8738282038863792, + "train_loss": 0.526295984530971, + "val_score": -1.7673837460024195 + }, + { + "epoch": 11, + "test_score": -1.9061960931879003, + "train_loss": 0.5257948676165003, + "val_score": -1.8011830309901071 + }, + { + "epoch": 12, + "test_score": -1.9085130234899337, + "train_loss": 0.5251704580592413, + "val_score": -1.8059021136263849 + }, + { + "epoch": 13, + "test_score": -1.9503421947808426, + "train_loss": 0.5246895671325879, + "val_score": -1.8436843731078842 + }, + { + "epoch": 14, + "test_score": -1.9205415643406698, + "train_loss": 0.5239885138334149, + "val_score": -1.809490599423118 + }, + { + "epoch": 15, + "test_score": -1.9227169642652717, + "train_loss": 0.5230375030179963, + "val_score": -1.8190888315355058 + }, + { + "epoch": 16, + "test_score": -1.90566319673516, + "train_loss": 0.5221649397463695, + "val_score": -1.7955265458730523 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.8738282038863792, + "score": -1.8738282038863792 + }, + "val": { + "rmse": 1.7673837460024195, + "score": -1.7673837460024195 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 59.403921604156494, + "variant": "rla_only" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1.toml new file mode 100644 index 000000000..bd2c0266b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1.toml @@ -0,0 +1,30 @@ +dataset = "delivery-eta" +variant = "rla_only" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1/report.json new file mode 100644 index 000000000..f59b665db --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1/report.json @@ -0,0 +1,113 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 2, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "rla_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.9529116094366537, + "train_loss": 0.626599464973394, + "val_score": -1.854167409603813 + }, + { + "epoch": 1, + "test_score": -1.9045921996561277, + "train_loss": 0.5439584403142442, + "val_score": -1.8019199039978397 + }, + { + "epoch": 2, + "test_score": -1.8829804025076913, + "train_loss": 0.5388047882240184, + "val_score": -1.7775892151066777 + }, + { + "epoch": 3, + "test_score": -1.917166050524368, + "train_loss": 0.5356628246551013, + "val_score": -1.8086206421799282 + }, + { + "epoch": 4, + "test_score": -1.8981720538645561, + "train_loss": 0.533457537854675, + "val_score": -1.7923243742261243 + }, + { + "epoch": 5, + "test_score": -1.9191541562961174, + "train_loss": 0.5313687770471086, + "val_score": -1.8099570357771626 + }, + { + "epoch": 6, + "test_score": -1.9167533177677576, + "train_loss": 0.530146632533874, + "val_score": -1.8146295205102208 + }, + { + "epoch": 7, + "test_score": -1.92939379229104, + "train_loss": 0.5287531240143045, + "val_score": -1.8220407034371349 + }, + { + "epoch": 8, + "test_score": -1.9524148489228184, + "train_loss": 0.5275940823293951, + "val_score": -1.8425391795781818 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.8829804025076913, + "score": -1.8829804025076913 + }, + "val": { + "rmse": 1.7775892151066777, + "score": -1.7775892151066777 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 32.074143171310425, + "variant": "rla_only" +} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2.toml new file mode 100644 index 000000000..b3419b31e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2.toml @@ -0,0 +1,30 @@ +dataset = "delivery-eta" +variant = "rla_only" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/delivery-eta" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2/report.json new file mode 100644 index 000000000..f3409a89f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2/report.json @@ -0,0 +1,101 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/delivery-eta" + }, + "dataset": "delivery-eta", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "rla_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2.toml", + "dataset": "delivery-eta", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -1.88058796129553, + "train_loss": 0.6275782894044026, + "val_score": -1.7734446252352045 + }, + { + "epoch": 1, + "test_score": -1.9163067797704876, + "train_loss": 0.5442314702663978, + "val_score": -1.8092503849001196 + }, + { + "epoch": 2, + "test_score": -1.9333606926308549, + "train_loss": 0.5406341039351303, + "val_score": -1.8277433600207929 + }, + { + "epoch": 3, + "test_score": -1.892036415576302, + "train_loss": 0.5362807738519934, + "val_score": -1.7886593960062 + }, + { + "epoch": 4, + "test_score": -1.931961140978318, + "train_loss": 0.5344498851438508, + "val_score": -1.8238907382305147 + }, + { + "epoch": 5, + "test_score": -1.8943497718331832, + "train_loss": 0.5325871710359615, + "val_score": -1.7891810348755857 + }, + { + "epoch": 6, + "test_score": -1.9185911206002868, + "train_loss": 0.5309783061490442, + "val_score": -1.8074469009556464 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 1.88058796129553, + "score": -1.88058796129553 + }, + "val": { + "rmse": 1.7734446252352045, + "score": -1.7734446252352045 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 26.989912509918213, + "variant": "rla_only" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..c91474fcb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0.toml @@ -0,0 +1,35 @@ +dataset = "ecom-offers" +variant = "all_four_combined" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..242fa0a59 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0/report.json @@ -0,0 +1,184 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 19, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_four_combined" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5681271222366269, + "train_loss": 0.5521719566097966, + "val_score": 0.5900086445530405 + }, + { + "epoch": 1, + "test_score": 0.5666844169705509, + "train_loss": 0.5123001651631461, + "val_score": 0.5863801419866583 + }, + { + "epoch": 2, + "test_score": 0.5681515585303227, + "train_loss": 0.5097376847708667, + "val_score": 0.588203584427758 + }, + { + "epoch": 3, + "test_score": 0.5695960057036864, + "train_loss": 0.507381233352202, + "val_score": 0.5914278861087139 + }, + { + "epoch": 4, + "test_score": 0.5722711537685954, + "train_loss": 0.5049635193966053, + "val_score": 0.5967446024280252 + }, + { + "epoch": 5, + "test_score": 0.5690483835267741, + "train_loss": 0.5053603251775106, + "val_score": 0.5903752258153991 + }, + { + "epoch": 6, + "test_score": 0.5708569203180814, + "train_loss": 0.5041680694730194, + "val_score": 0.5948900154498427 + }, + { + "epoch": 7, + "test_score": 0.5712752012296551, + "train_loss": 0.5012796890956385, + "val_score": 0.5947254295989219 + }, + { + "epoch": 8, + "test_score": 0.5715983591258145, + "train_loss": 0.5017772890903331, + "val_score": 0.5956531174159587 + }, + { + "epoch": 9, + "test_score": 0.5696216228989421, + "train_loss": 0.5025770465532938, + "val_score": 0.5918668799860465 + }, + { + "epoch": 10, + "test_score": 0.5728374410090079, + "train_loss": 0.5002644200015951, + "val_score": 0.5988367627801356 + }, + { + "epoch": 11, + "test_score": 0.5718931908819324, + "train_loss": 0.5004737333015159, + "val_score": 0.5952545992753181 + }, + { + "epoch": 12, + "test_score": 0.5711061773326337, + "train_loss": 0.4999077397364157, + "val_score": 0.593878350521634 + }, + { + "epoch": 13, + "test_score": 0.5740794395020907, + "train_loss": 0.49866839636255195, + "val_score": 0.5986739163945447 + }, + { + "epoch": 14, + "test_score": 0.5721571115323896, + "train_loss": 0.5007636806479207, + "val_score": 0.5954374826799741 + }, + { + "epoch": 15, + "test_score": 0.5738696419567221, + "train_loss": 0.4993751821694551, + "val_score": 0.5991782286691394 + }, + { + "epoch": 16, + "test_score": 0.5739100870602782, + "train_loss": 0.4982121962088126, + "val_score": 0.5991561942131909 + }, + { + "epoch": 17, + "test_score": 0.5756130369945278, + "train_loss": 0.4973638112898226, + "val_score": 0.6010971648086056 + }, + { + "epoch": 18, + "test_score": 0.5745054846089597, + "train_loss": 0.496809223735774, + "val_score": 0.5997656556928566 + }, + { + "epoch": 19, + "test_score": 0.575590289916974, + "train_loss": 0.49563960841408483, + "val_score": 0.6024052943310221 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.575590289916974, + "score": 0.575590289916974 + }, + "val": { + "roc-auc": 0.6024052943310221, + "score": 0.6024052943310221 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0", + "seed": 0, + "task_type": "binclass", + "time_seconds": 35.4436354637146, + "variant": "all_four_combined" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1.toml new file mode 100644 index 000000000..48180f642 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1.toml @@ -0,0 +1,35 @@ +dataset = "ecom-offers" +variant = "all_four_combined" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1/report.json new file mode 100644 index 000000000..c664ae19a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1/report.json @@ -0,0 +1,184 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 17, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_four_combined" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.570155176539948, + "train_loss": 0.5530174418731972, + "val_score": 0.5931137338939699 + }, + { + "epoch": 1, + "test_score": 0.5710579773322195, + "train_loss": 0.5131659612611488, + "val_score": 0.5945566940913031 + }, + { + "epoch": 2, + "test_score": 0.5710214871640712, + "train_loss": 0.5079680477027539, + "val_score": 0.5940891611895743 + }, + { + "epoch": 3, + "test_score": 0.5690187556054855, + "train_loss": 0.5082374629047182, + "val_score": 0.5892378108109451 + }, + { + "epoch": 4, + "test_score": 0.5692918414158183, + "train_loss": 0.5057879333142881, + "val_score": 0.5899993280946632 + }, + { + "epoch": 5, + "test_score": 0.5717230294760591, + "train_loss": 0.5025530799671456, + "val_score": 0.5962786947470815 + }, + { + "epoch": 6, + "test_score": 0.5718020382993217, + "train_loss": 0.5034321209898701, + "val_score": 0.59612584660183 + }, + { + "epoch": 7, + "test_score": 0.5729451634072069, + "train_loss": 0.5040367698227918, + "val_score": 0.5965978682085837 + }, + { + "epoch": 8, + "test_score": 0.5713202769550761, + "train_loss": 0.5016454839044147, + "val_score": 0.5942215448200439 + }, + { + "epoch": 9, + "test_score": 0.5705815532910871, + "train_loss": 0.5025773329867257, + "val_score": 0.5937047577781998 + }, + { + "epoch": 10, + "test_score": 0.572931395520779, + "train_loss": 0.5005058971820054, + "val_score": 0.5959005858416042 + }, + { + "epoch": 11, + "test_score": 0.5737611477882123, + "train_loss": 0.49932017315317084, + "val_score": 0.5962125066171546 + }, + { + "epoch": 12, + "test_score": 0.5760470260664834, + "train_loss": 0.4990245197658186, + "val_score": 0.5994599151786585 + }, + { + "epoch": 13, + "test_score": 0.5736891096935801, + "train_loss": 0.4994683022852297, + "val_score": 0.5968734702747561 + }, + { + "epoch": 14, + "test_score": 0.5759794977140287, + "train_loss": 0.4970110097417125, + "val_score": 0.6010394307750069 + }, + { + "epoch": 15, + "test_score": 0.5753586999336326, + "train_loss": 0.49861614792435255, + "val_score": 0.5993183809286673 + }, + { + "epoch": 16, + "test_score": 0.5749285542158353, + "train_loss": 0.4977775944603814, + "val_score": 0.5981364732019077 + }, + { + "epoch": 17, + "test_score": 0.5770100559377882, + "train_loss": 0.4978238934720004, + "val_score": 0.6032831567852186 + }, + { + "epoch": 18, + "test_score": 0.575521803825459, + "train_loss": 0.4963515991414035, + "val_score": 0.6008931203659741 + }, + { + "epoch": 19, + "test_score": 0.5767410428430715, + "train_loss": 0.4981405012033604, + "val_score": 0.6010097161373898 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5770100559377882, + "score": 0.5770100559377882 + }, + "val": { + "roc-auc": 0.6032831567852186, + "score": 0.6032831567852186 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1", + "seed": 1, + "task_type": "binclass", + "time_seconds": 36.04194712638855, + "variant": "all_four_combined" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2.toml new file mode 100644 index 000000000..95d814f71 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2.toml @@ -0,0 +1,35 @@ +dataset = "ecom-offers" +variant = "all_four_combined" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2/report.json new file mode 100644 index 000000000..d3c459911 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2/report.json @@ -0,0 +1,112 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 1, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_four_combined" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5662744736579172, + "train_loss": 0.5533291416035758, + "val_score": 0.5870080889707623 + }, + { + "epoch": 1, + "test_score": 0.5734697781503159, + "train_loss": 0.5128997222140983, + "val_score": 0.5995627021013641 + }, + { + "epoch": 2, + "test_score": 0.5708813690096939, + "train_loss": 0.5101203283777943, + "val_score": 0.5947186744295367 + }, + { + "epoch": 3, + "test_score": 0.572774698252376, + "train_loss": 0.5073363532622656, + "val_score": 0.5972117999667349 + }, + { + "epoch": 4, + "test_score": 0.569280137782511, + "train_loss": 0.5072464628352059, + "val_score": 0.5914310996972048 + }, + { + "epoch": 5, + "test_score": 0.568470256275971, + "train_loss": 0.5035725958921291, + "val_score": 0.5907585457462503 + }, + { + "epoch": 6, + "test_score": 0.570999490160474, + "train_loss": 0.5035661116794303, + "val_score": 0.5938800347073453 + }, + { + "epoch": 7, + "test_score": 0.5664802511788156, + "train_loss": 0.5036690522123266, + "val_score": 0.5869948255476216 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5734697781503159, + "score": 0.5734697781503159 + }, + "val": { + "roc-auc": 0.5995627021013641, + "score": 0.5995627021013641 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2", + "seed": 2, + "task_type": "binclass", + "time_seconds": 15.193928480148315, + "variant": "all_four_combined" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..df11e7b0d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0.toml @@ -0,0 +1,34 @@ +dataset = "ecom-offers" +variant = "all_minus_cf_fisd" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..79a9171e3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0/report.json @@ -0,0 +1,105 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_minus_cf_fisd" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.574504443183962, + "train_loss": 0.5605815064024042, + "val_score": 0.6001605290639818 + }, + { + "epoch": 1, + "test_score": 0.5624732538194603, + "train_loss": 0.5235076811578538, + "val_score": 0.5808439434699122 + }, + { + "epoch": 2, + "test_score": 0.5659337572123175, + "train_loss": 0.5128068581775382, + "val_score": 0.5866805240631663 + }, + { + "epoch": 3, + "test_score": 0.5697632009080036, + "train_loss": 0.5094030946493149, + "val_score": 0.5926678706027072 + }, + { + "epoch": 4, + "test_score": 0.5736462779910708, + "train_loss": 0.5063647743728426, + "val_score": 0.5980532736907034 + }, + { + "epoch": 5, + "test_score": 0.5719072191246096, + "train_loss": 0.5063130502347593, + "val_score": 0.5954887711100517 + }, + { + "epoch": 6, + "test_score": 0.5747901686703085, + "train_loss": 0.5049416814689283, + "val_score": 0.5994671346968392 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.574504443183962, + "score": 0.574504443183962 + }, + "val": { + "roc-auc": 0.6001605290639818, + "score": 0.6001605290639818 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0", + "seed": 0, + "task_type": "binclass", + "time_seconds": 12.681254625320435, + "variant": "all_minus_cf_fisd" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1.toml new file mode 100644 index 000000000..39def5ff8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1.toml @@ -0,0 +1,34 @@ +dataset = "ecom-offers" +variant = "all_minus_cf_fisd" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1/report.json new file mode 100644 index 000000000..3bef0e355 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1/report.json @@ -0,0 +1,105 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_minus_cf_fisd" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5749947559912094, + "train_loss": 0.5615878336959415, + "val_score": 0.6007862316955732 + }, + { + "epoch": 1, + "test_score": 0.5660149852626585, + "train_loss": 0.5236683112603647, + "val_score": 0.5866883184893799 + }, + { + "epoch": 2, + "test_score": 0.5685566759539047, + "train_loss": 0.5110080148334857, + "val_score": 0.5900513793834766 + }, + { + "epoch": 3, + "test_score": 0.5673698016976789, + "train_loss": 0.5102973282337189, + "val_score": 0.587963685624544 + }, + { + "epoch": 4, + "test_score": 0.5698586431697714, + "train_loss": 0.5082294896796897, + "val_score": 0.591891214074126 + }, + { + "epoch": 5, + "test_score": 0.5750190621067808, + "train_loss": 0.5046696767762855, + "val_score": 0.5987031814245977 + }, + { + "epoch": 6, + "test_score": 0.5755671523050461, + "train_loss": 0.5048941142029233, + "val_score": 0.6002077083757481 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5749947559912094, + "score": 0.5749947559912094 + }, + "val": { + "roc-auc": 0.6007862316955732, + "score": 0.6007862316955732 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1", + "seed": 1, + "task_type": "binclass", + "time_seconds": 13.13693618774414, + "variant": "all_minus_cf_fisd" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2.toml new file mode 100644 index 000000000..53cab0f8d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2.toml @@ -0,0 +1,34 @@ +dataset = "ecom-offers" +variant = "all_minus_cf_fisd" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2/report.json new file mode 100644 index 000000000..bc6324e94 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2/report.json @@ -0,0 +1,183 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 19, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_minus_cf_fisd" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.567753504819746, + "train_loss": 0.5610457912639335, + "val_score": 0.5919127657547392 + }, + { + "epoch": 1, + "test_score": 0.5677904785066432, + "train_loss": 0.5248338960938983, + "val_score": 0.5898634213098389 + }, + { + "epoch": 2, + "test_score": 0.56833937456949, + "train_loss": 0.5131386976551127, + "val_score": 0.5908690386477797 + }, + { + "epoch": 3, + "test_score": 0.5722036750078073, + "train_loss": 0.5094726416799757, + "val_score": 0.5954165538163965 + }, + { + "epoch": 4, + "test_score": 0.5688222672236289, + "train_loss": 0.5084855302616402, + "val_score": 0.5919105324581503 + }, + { + "epoch": 5, + "test_score": 0.5723461921590541, + "train_loss": 0.504546512608175, + "val_score": 0.5960148524984257 + }, + { + "epoch": 6, + "test_score": 0.5740603312130705, + "train_loss": 0.5045429435041215, + "val_score": 0.5995567245319464 + }, + { + "epoch": 7, + "test_score": 0.5703335918588237, + "train_loss": 0.504522568097821, + "val_score": 0.593099910304028 + }, + { + "epoch": 8, + "test_score": 0.5740792566328203, + "train_loss": 0.502777013513777, + "val_score": 0.5989479485195507 + }, + { + "epoch": 9, + "test_score": 0.5768869384266024, + "train_loss": 0.5016946373162446, + "val_score": 0.6014101764352952 + }, + { + "epoch": 10, + "test_score": 0.576792432207541, + "train_loss": 0.5012201556452999, + "val_score": 0.602760535900965 + }, + { + "epoch": 11, + "test_score": 0.5777785160048363, + "train_loss": 0.5014492196065409, + "val_score": 0.6048366430518524 + }, + { + "epoch": 12, + "test_score": 0.5750577993973199, + "train_loss": 0.4990316596296098, + "val_score": 0.6011327059180163 + }, + { + "epoch": 13, + "test_score": 0.5756340421647939, + "train_loss": 0.4997640174848062, + "val_score": 0.601904512581398 + }, + { + "epoch": 14, + "test_score": 0.5750410467124611, + "train_loss": 0.4982257490908658, + "val_score": 0.6003227636884607 + }, + { + "epoch": 15, + "test_score": 0.5774879212367299, + "train_loss": 0.49788987912513594, + "val_score": 0.6036308139914984 + }, + { + "epoch": 16, + "test_score": 0.5785781506332739, + "train_loss": 0.4980563951863183, + "val_score": 0.6051532294272071 + }, + { + "epoch": 17, + "test_score": 0.5771495046046669, + "train_loss": 0.49732758656696036, + "val_score": 0.6033888071921645 + }, + { + "epoch": 18, + "test_score": 0.577803860445926, + "train_loss": 0.49810293592788557, + "val_score": 0.6039634204003048 + }, + { + "epoch": 19, + "test_score": 0.5807682519060278, + "train_loss": 0.49806651362666376, + "val_score": 0.6077083676404789 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5807682519060278, + "score": 0.5807682519060278 + }, + "val": { + "roc-auc": 0.6077083676404789, + "score": 0.6077083676404789 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2", + "seed": 2, + "task_type": "binclass", + "time_seconds": 37.24916982650757, + "variant": "all_minus_cf_fisd" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0.toml new file mode 100644 index 000000000..05e9af8f5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0.toml @@ -0,0 +1,33 @@ +dataset = "ecom-offers" +variant = "all_minus_esam" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0/report.json new file mode 100644 index 000000000..80ec1d7b3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0/report.json @@ -0,0 +1,140 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 6, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_minus_esam" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5684889616327005, + "train_loss": 0.5520850326176043, + "val_score": 0.5907757045398437 + }, + { + "epoch": 1, + "test_score": 0.570941281941853, + "train_loss": 0.5112152778440051, + "val_score": 0.594774130942852 + }, + { + "epoch": 2, + "test_score": 0.5696106662401121, + "train_loss": 0.5086778380252697, + "val_score": 0.5913595605002036 + }, + { + "epoch": 3, + "test_score": 0.569436819553518, + "train_loss": 0.5053646884582661, + "val_score": 0.5906331420889478 + }, + { + "epoch": 4, + "test_score": 0.5703592555462669, + "train_loss": 0.5050829985627422, + "val_score": 0.5940241044491734 + }, + { + "epoch": 5, + "test_score": 0.5697211688711175, + "train_loss": 0.5059763888518015, + "val_score": 0.5918137120489388 + }, + { + "epoch": 6, + "test_score": 0.573248515631535, + "train_loss": 0.5028402137535589, + "val_score": 0.5978648475829621 + }, + { + "epoch": 7, + "test_score": 0.5716950039854963, + "train_loss": 0.5033861282798979, + "val_score": 0.5947317756991297 + }, + { + "epoch": 8, + "test_score": 0.5701298289993791, + "train_loss": 0.501969211631351, + "val_score": 0.5932625687389158 + }, + { + "epoch": 9, + "test_score": 0.5698557916489444, + "train_loss": 0.501433088823601, + "val_score": 0.5918341913049523 + }, + { + "epoch": 10, + "test_score": 0.5706267467967163, + "train_loss": 0.5014634259321071, + "val_score": 0.594032311629872 + }, + { + "epoch": 11, + "test_score": 0.5717393451343564, + "train_loss": 0.4991199186554662, + "val_score": 0.5947687946170093 + }, + { + "epoch": 12, + "test_score": 0.5723682852465053, + "train_loss": 0.49935837365962843, + "val_score": 0.5952143188599449 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.573248515631535, + "score": 0.573248515631535 + }, + "val": { + "roc-auc": 0.5978648475829621, + "score": 0.5978648475829621 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0", + "seed": 0, + "task_type": "binclass", + "time_seconds": 17.78662633895874, + "variant": "all_minus_esam" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1.toml new file mode 100644 index 000000000..35f230039 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1.toml @@ -0,0 +1,33 @@ +dataset = "ecom-offers" +variant = "all_minus_esam" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1/report.json new file mode 100644 index 000000000..924afcaeb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1/report.json @@ -0,0 +1,110 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 1, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_minus_esam" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5696132883994813, + "train_loss": 0.5533099483560633, + "val_score": 0.5920076476919942 + }, + { + "epoch": 1, + "test_score": 0.5715625167488105, + "train_loss": 0.5137152550397096, + "val_score": 0.5950530313595969 + }, + { + "epoch": 2, + "test_score": 0.5694419026993401, + "train_loss": 0.5094894523973819, + "val_score": 0.5910633723040886 + }, + { + "epoch": 3, + "test_score": 0.5706710197570348, + "train_loss": 0.507937710042353, + "val_score": 0.5929211286470673 + }, + { + "epoch": 4, + "test_score": 0.5702061660716059, + "train_loss": 0.5054546721555568, + "val_score": 0.591947013321076 + }, + { + "epoch": 5, + "test_score": 0.5686125905579476, + "train_loss": 0.5028857384566907, + "val_score": 0.5881382991982672 + }, + { + "epoch": 6, + "test_score": 0.5691999232618552, + "train_loss": 0.5034620254128067, + "val_score": 0.5910225059247589 + }, + { + "epoch": 7, + "test_score": 0.571275508078092, + "train_loss": 0.5015709935515015, + "val_score": 0.5946880679474075 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5715625167488105, + "score": 0.5715625167488105 + }, + "val": { + "roc-auc": 0.5950530313595969, + "score": 0.5950530313595969 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1", + "seed": 1, + "task_type": "binclass", + "time_seconds": 11.00665020942688, + "variant": "all_minus_esam" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2.toml new file mode 100644 index 000000000..f5f0efcd2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2.toml @@ -0,0 +1,33 @@ +dataset = "ecom-offers" +variant = "all_minus_esam" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2/report.json new file mode 100644 index 000000000..350987a4c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2/report.json @@ -0,0 +1,110 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 1, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_minus_esam" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5665558350776065, + "train_loss": 0.5528011929105829, + "val_score": 0.5871859787832107 + }, + { + "epoch": 1, + "test_score": 0.5712034110933558, + "train_loss": 0.5117590449474476, + "val_score": 0.5952215125809704 + }, + { + "epoch": 2, + "test_score": 0.5690808288746189, + "train_loss": 0.5099736430026867, + "val_score": 0.5909409390049554 + }, + { + "epoch": 3, + "test_score": 0.5692677243684758, + "train_loss": 0.5080657518572278, + "val_score": 0.590635250085068 + }, + { + "epoch": 4, + "test_score": 0.570682118991906, + "train_loss": 0.5076316021106861, + "val_score": 0.5936173017363682 + }, + { + "epoch": 5, + "test_score": 0.5677198320781538, + "train_loss": 0.5045595886530699, + "val_score": 0.5892704073593417 + }, + { + "epoch": 6, + "test_score": 0.5699361952378288, + "train_loss": 0.5028332068964287, + "val_score": 0.5939281943110137 + }, + { + "epoch": 7, + "test_score": 0.5693752638974043, + "train_loss": 0.5033974564737744, + "val_score": 0.5922146514368714 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5712034110933558, + "score": 0.5712034110933558 + }, + "val": { + "roc-auc": 0.5952215125809704, + "score": 0.5952215125809704 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2", + "seed": 2, + "task_type": "binclass", + "time_seconds": 12.06042766571045, + "variant": "all_minus_esam" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0.toml new file mode 100644 index 000000000..64e91ab4e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0.toml @@ -0,0 +1,33 @@ +dataset = "ecom-offers" +variant = "all_minus_mfb" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0/report.json new file mode 100644 index 000000000..ae52fbade --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0/report.json @@ -0,0 +1,140 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 6, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_minus_mfb" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5674570582349857, + "train_loss": 0.5495740638838874, + "val_score": 0.5886585062058447 + }, + { + "epoch": 1, + "test_score": 0.5725468989320472, + "train_loss": 0.5061508588216923, + "val_score": 0.5971063927901108 + }, + { + "epoch": 2, + "test_score": 0.5728618866011412, + "train_loss": 0.502218743717229, + "val_score": 0.5970249769679312 + }, + { + "epoch": 3, + "test_score": 0.5734975432846294, + "train_loss": 0.4995915039821907, + "val_score": 0.599563420736405 + }, + { + "epoch": 4, + "test_score": 0.5742185007340744, + "train_loss": 0.4979530843319716, + "val_score": 0.6018211546019538 + }, + { + "epoch": 5, + "test_score": 0.5739961750939417, + "train_loss": 0.49669913450876874, + "val_score": 0.6017292503934841 + }, + { + "epoch": 6, + "test_score": 0.5743883676894275, + "train_loss": 0.49562931612685873, + "val_score": 0.6039636304628554 + }, + { + "epoch": 7, + "test_score": 0.5742671656563625, + "train_loss": 0.4947642540490186, + "val_score": 0.603617392100118 + }, + { + "epoch": 8, + "test_score": 0.5736554896431338, + "train_loss": 0.4941260549757216, + "val_score": 0.60284559649266 + }, + { + "epoch": 9, + "test_score": 0.5724527801478807, + "train_loss": 0.4940890046181502, + "val_score": 0.600026829778591 + }, + { + "epoch": 10, + "test_score": 0.5729124050119669, + "train_loss": 0.493927875050792, + "val_score": 0.6011250257363477 + }, + { + "epoch": 11, + "test_score": 0.5726416531094415, + "train_loss": 0.49346746117980395, + "val_score": 0.6002735795691954 + }, + { + "epoch": 12, + "test_score": 0.5719451319536923, + "train_loss": 0.49327058096726734, + "val_score": 0.5997216752283494 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5743883676894275, + "score": 0.5743883676894275 + }, + "val": { + "roc-auc": 0.6039636304628554, + "score": 0.6039636304628554 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0", + "seed": 0, + "task_type": "binclass", + "time_seconds": 23.100914239883423, + "variant": "all_minus_mfb" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1.toml new file mode 100644 index 000000000..71fdb2d08 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1.toml @@ -0,0 +1,33 @@ +dataset = "ecom-offers" +variant = "all_minus_mfb" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1/report.json new file mode 100644 index 000000000..f3b0be255 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1/report.json @@ -0,0 +1,146 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 7, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_minus_mfb" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5698811081947217, + "train_loss": 0.5510928448703554, + "val_score": 0.592789832182429 + }, + { + "epoch": 1, + "test_score": 0.5727224999238458, + "train_loss": 0.5068844297417888, + "val_score": 0.5969781735575709 + }, + { + "epoch": 2, + "test_score": 0.5736512681525181, + "train_loss": 0.5025383046379795, + "val_score": 0.5984598294878791 + }, + { + "epoch": 3, + "test_score": 0.5738685013483913, + "train_loss": 0.5001370167290723, + "val_score": 0.6001265910635086 + }, + { + "epoch": 4, + "test_score": 0.5723961495641519, + "train_loss": 0.4980512509743373, + "val_score": 0.5985286194451851 + }, + { + "epoch": 5, + "test_score": 0.5721636700302917, + "train_loss": 0.49637439515855575, + "val_score": 0.598715663562463 + }, + { + "epoch": 6, + "test_score": 0.5737822366444156, + "train_loss": 0.4955915207112277, + "val_score": 0.6027431301917421 + }, + { + "epoch": 7, + "test_score": 0.5738892213665745, + "train_loss": 0.4948982331487868, + "val_score": 0.6032786091152671 + }, + { + "epoch": 8, + "test_score": 0.5712110668068804, + "train_loss": 0.4943665370897011, + "val_score": 0.5972855982574803 + }, + { + "epoch": 9, + "test_score": 0.5718640464793927, + "train_loss": 0.4937717644152818, + "val_score": 0.5991909429814026 + }, + { + "epoch": 10, + "test_score": 0.5725859089767524, + "train_loss": 0.493560039334827, + "val_score": 0.6007264781132439 + }, + { + "epoch": 11, + "test_score": 0.5725165829263852, + "train_loss": 0.49275064689141734, + "val_score": 0.5994298578074065 + }, + { + "epoch": 12, + "test_score": 0.5718745909074944, + "train_loss": 0.49317565118824996, + "val_score": 0.598923736046632 + }, + { + "epoch": 13, + "test_score": 0.5721781352995304, + "train_loss": 0.4925886651983968, + "val_score": 0.5997260238916742 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5738892213665745, + "score": 0.5738892213665745 + }, + "val": { + "roc-auc": 0.6032786091152671, + "score": 0.6032786091152671 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1", + "seed": 1, + "task_type": "binclass", + "time_seconds": 26.137808322906494, + "variant": "all_minus_mfb" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2.toml new file mode 100644 index 000000000..1224f97d7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2.toml @@ -0,0 +1,33 @@ +dataset = "ecom-offers" +variant = "all_minus_mfb" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2/report.json new file mode 100644 index 000000000..affbd78ac --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2/report.json @@ -0,0 +1,122 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 3, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_minus_mfb" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5680833018002656, + "train_loss": 0.5508555427745536, + "val_score": 0.5904751455685565 + }, + { + "epoch": 1, + "test_score": 0.5738387680448112, + "train_loss": 0.5068491857360911, + "val_score": 0.6003150761361764 + }, + { + "epoch": 2, + "test_score": 0.5735754703896652, + "train_loss": 0.5023968407401332, + "val_score": 0.5990004420453122 + }, + { + "epoch": 3, + "test_score": 0.5748292097098053, + "train_loss": 0.49968371843850173, + "val_score": 0.6013164627406452 + }, + { + "epoch": 4, + "test_score": 0.5742371472006997, + "train_loss": 0.4976544071126867, + "val_score": 0.6012733298969557 + }, + { + "epoch": 5, + "test_score": 0.5717932171816329, + "train_loss": 0.495533820103716, + "val_score": 0.5970501918445993 + }, + { + "epoch": 6, + "test_score": 0.5729961312425107, + "train_loss": 0.49507507902604564, + "val_score": 0.5994206887613455 + }, + { + "epoch": 7, + "test_score": 0.5724098802568298, + "train_loss": 0.4942484376607118, + "val_score": 0.5982748491430266 + }, + { + "epoch": 8, + "test_score": 0.5712021000136711, + "train_loss": 0.493870695983922, + "val_score": 0.5960575541610907 + }, + { + "epoch": 9, + "test_score": 0.5722020694776025, + "train_loss": 0.49397959532561125, + "val_score": 0.5977455652220854 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5748292097098053, + "score": 0.5748292097098053 + }, + "val": { + "roc-auc": 0.6013164627406452, + "score": 0.6013164627406452 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2", + "seed": 2, + "task_type": "binclass", + "time_seconds": 19.141494274139404, + "variant": "all_minus_mfb" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0.toml new file mode 100644 index 000000000..81cade18c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0.toml @@ -0,0 +1,32 @@ +dataset = "ecom-offers" +variant = "all_minus_rla" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0/report.json new file mode 100644 index 000000000..0faafa64d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0/report.json @@ -0,0 +1,103 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_minus_rla" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5697199786711201, + "train_loss": 0.5528302910151305, + "val_score": 0.5933440803791009 + }, + { + "epoch": 1, + "test_score": 0.5666581674815466, + "train_loss": 0.5124878894399714, + "val_score": 0.5865858374472304 + }, + { + "epoch": 2, + "test_score": 0.5704257579711196, + "train_loss": 0.5099211677357003, + "val_score": 0.5922768410077242 + }, + { + "epoch": 3, + "test_score": 0.5701217269608554, + "train_loss": 0.5068588825287642, + "val_score": 0.5924421676055333 + }, + { + "epoch": 4, + "test_score": 0.5699790982283588, + "train_loss": 0.5055394018137896, + "val_score": 0.5924149442360582 + }, + { + "epoch": 5, + "test_score": 0.5687964392637903, + "train_loss": 0.506475669366342, + "val_score": 0.5901035006931401 + }, + { + "epoch": 6, + "test_score": 0.569307100151722, + "train_loss": 0.5051003059855214, + "val_score": 0.5923733334245325 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5697199786711201, + "score": 0.5697199786711201 + }, + "val": { + "roc-auc": 0.5933440803791009, + "score": 0.5933440803791009 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0", + "seed": 0, + "task_type": "binclass", + "time_seconds": 11.543763399124146, + "variant": "all_minus_rla" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1.toml new file mode 100644 index 000000000..9fee2aa4d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1.toml @@ -0,0 +1,32 @@ +dataset = "ecom-offers" +variant = "all_minus_rla" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1/report.json new file mode 100644 index 000000000..6f687dd89 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1/report.json @@ -0,0 +1,133 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 5, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_minus_rla" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5709148433846198, + "train_loss": 0.5527302070900246, + "val_score": 0.5948060788770578 + }, + { + "epoch": 1, + "test_score": 0.569859216573416, + "train_loss": 0.5132867015070386, + "val_score": 0.5924314986391559 + }, + { + "epoch": 2, + "test_score": 0.5704153778154134, + "train_loss": 0.5095344671496639, + "val_score": 0.5928492098633522 + }, + { + "epoch": 3, + "test_score": 0.5686327836646736, + "train_loss": 0.5084150687411979, + "val_score": 0.5885501692094383 + }, + { + "epoch": 4, + "test_score": 0.5708701612930521, + "train_loss": 0.5059906867919145, + "val_score": 0.5926585762561773 + }, + { + "epoch": 5, + "test_score": 0.573474359180514, + "train_loss": 0.5031700454376362, + "val_score": 0.5984686926533844 + }, + { + "epoch": 6, + "test_score": 0.5729537334670838, + "train_loss": 0.5041169088195872, + "val_score": 0.5971772428345342 + }, + { + "epoch": 7, + "test_score": 0.5720949793731522, + "train_loss": 0.5038585844967101, + "val_score": 0.5956662702798624 + }, + { + "epoch": 8, + "test_score": 0.5696481420426334, + "train_loss": 0.5019371962105786, + "val_score": 0.591666911493955 + }, + { + "epoch": 9, + "test_score": 0.5705172298000838, + "train_loss": 0.503837333785163, + "val_score": 0.5938676336462538 + }, + { + "epoch": 10, + "test_score": 0.5720236944519559, + "train_loss": 0.5018296633605603, + "val_score": 0.5940563214108562 + }, + { + "epoch": 11, + "test_score": 0.5724766182420986, + "train_loss": 0.5006160101404896, + "val_score": 0.5938442135145341 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.573474359180514, + "score": 0.573474359180514 + }, + "val": { + "roc-auc": 0.5984686926533844, + "score": 0.5984686926533844 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1", + "seed": 1, + "task_type": "binclass", + "time_seconds": 19.83189105987549, + "variant": "all_minus_rla" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2.toml new file mode 100644 index 000000000..2be994558 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2.toml @@ -0,0 +1,32 @@ +dataset = "ecom-offers" +variant = "all_minus_rla" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2/report.json new file mode 100644 index 000000000..0b806f5b4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2/report.json @@ -0,0 +1,109 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 1, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_minus_rla" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5640513629321112, + "train_loss": 0.5529496222734451, + "val_score": 0.5826846405226108 + }, + { + "epoch": 1, + "test_score": 0.5729159973083131, + "train_loss": 0.5133549846984722, + "val_score": 0.5991167798451749 + }, + { + "epoch": 2, + "test_score": 0.5704932894230534, + "train_loss": 0.5109604150056839, + "val_score": 0.594155806297681 + }, + { + "epoch": 3, + "test_score": 0.571275858319237, + "train_loss": 0.5076655331585143, + "val_score": 0.5947095790896337 + }, + { + "epoch": 4, + "test_score": 0.5672518014265862, + "train_loss": 0.5066016233629651, + "val_score": 0.5872497604070774 + }, + { + "epoch": 5, + "test_score": 0.5696887049263977, + "train_loss": 0.5043161257549569, + "val_score": 0.5932295852331895 + }, + { + "epoch": 6, + "test_score": 0.5696682948561305, + "train_loss": 0.5044200701846017, + "val_score": 0.5937620274630029 + }, + { + "epoch": 7, + "test_score": 0.5669393181366531, + "train_loss": 0.5049267328447766, + "val_score": 0.5873475905906542 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5729159973083131, + "score": 0.5729159973083131 + }, + "val": { + "roc-auc": 0.5991167798451749, + "score": 0.5991167798451749 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2", + "seed": 2, + "task_type": "binclass", + "time_seconds": 13.887128829956055, + "variant": "all_minus_rla" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..6624bf9a2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0.toml @@ -0,0 +1,27 @@ +dataset = "ecom-offers" +variant = "baseline_plr" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..41fd7246b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0/report.json @@ -0,0 +1,176 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 14, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "baseline_plr" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5750006171063008, + "train_loss": 0.5606814733258, + "val_score": 0.601598805234971 + }, + { + "epoch": 1, + "test_score": 0.5678999118173383, + "train_loss": 0.5204960731444536, + "val_score": 0.5896181788101665 + }, + { + "epoch": 2, + "test_score": 0.5716925058052935, + "train_loss": 0.506363250039242, + "val_score": 0.5955156628018154 + }, + { + "epoch": 3, + "test_score": 0.5729560239821828, + "train_loss": 0.501323691672749, + "val_score": 0.5973503675438647 + }, + { + "epoch": 4, + "test_score": 0.5743658530728106, + "train_loss": 0.49991664235238675, + "val_score": 0.5996836539067204 + }, + { + "epoch": 5, + "test_score": 0.5756228344481519, + "train_loss": 0.49864044068036256, + "val_score": 0.6016151643167498 + }, + { + "epoch": 6, + "test_score": 0.5758462325080738, + "train_loss": 0.4971861519195415, + "val_score": 0.602125303063138 + }, + { + "epoch": 7, + "test_score": 0.5772494969018412, + "train_loss": 0.4960625370343526, + "val_score": 0.6039556112328598 + }, + { + "epoch": 8, + "test_score": 0.5749955897511035, + "train_loss": 0.4952058858341641, + "val_score": 0.6014264618109157 + }, + { + "epoch": 9, + "test_score": 0.5751647407267706, + "train_loss": 0.4940291941165924, + "val_score": 0.6030069761257056 + }, + { + "epoch": 10, + "test_score": 0.5740187919933709, + "train_loss": 0.4936152751798983, + "val_score": 0.602776920779899 + }, + { + "epoch": 11, + "test_score": 0.5753113119967586, + "train_loss": 0.49395177761713666, + "val_score": 0.6049119965425326 + }, + { + "epoch": 12, + "test_score": 0.5730560534731072, + "train_loss": 0.4931856112347709, + "val_score": 0.602283498590156 + }, + { + "epoch": 13, + "test_score": 0.5752409321234768, + "train_loss": 0.4932894745358714, + "val_score": 0.6063850878172757 + }, + { + "epoch": 14, + "test_score": 0.5767959687132622, + "train_loss": 0.49289950397279525, + "val_score": 0.6083677687275297 + }, + { + "epoch": 15, + "test_score": 0.5761139159262311, + "train_loss": 0.49263357288307613, + "val_score": 0.607897663480885 + }, + { + "epoch": 16, + "test_score": 0.5763534684715339, + "train_loss": 0.4920575221379598, + "val_score": 0.6080092030098587 + }, + { + "epoch": 17, + "test_score": 0.5754996890416538, + "train_loss": 0.49198202733640317, + "val_score": 0.6067648698525406 + }, + { + "epoch": 18, + "test_score": 0.5760725502773644, + "train_loss": 0.49181015182424476, + "val_score": 0.6074467255206407 + }, + { + "epoch": 19, + "test_score": 0.5760798929434939, + "train_loss": 0.4918224579758114, + "val_score": 0.6080325310088809 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5767959687132622, + "score": 0.5767959687132622 + }, + "val": { + "roc-auc": 0.6083677687275297, + "score": 0.6083677687275297 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0", + "seed": 0, + "task_type": "binclass", + "time_seconds": 25.43160104751587, + "variant": "baseline_plr" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1.toml new file mode 100644 index 000000000..71cc0e428 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1.toml @@ -0,0 +1,27 @@ +dataset = "ecom-offers" +variant = "baseline_plr" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1/report.json new file mode 100644 index 000000000..8223ba01e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1/report.json @@ -0,0 +1,176 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 14, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "baseline_plr" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5746579324914347, + "train_loss": 0.5609680237593474, + "val_score": 0.6008386257180214 + }, + { + "epoch": 1, + "test_score": 0.5635432591144864, + "train_loss": 0.5211676916590443, + "val_score": 0.5821124669883017 + }, + { + "epoch": 2, + "test_score": 0.5713524464492756, + "train_loss": 0.5065342673548946, + "val_score": 0.5941694087691489 + }, + { + "epoch": 3, + "test_score": 0.5728148984970787, + "train_loss": 0.5021586578201365, + "val_score": 0.5963798159106152 + }, + { + "epoch": 4, + "test_score": 0.5745534273526035, + "train_loss": 0.49993850345964785, + "val_score": 0.5985960937475701 + }, + { + "epoch": 5, + "test_score": 0.5757297199869778, + "train_loss": 0.498760970654311, + "val_score": 0.6008029003432156 + }, + { + "epoch": 6, + "test_score": 0.5770634072725658, + "train_loss": 0.4972201293265378, + "val_score": 0.6031019059719633 + }, + { + "epoch": 7, + "test_score": 0.5774046103363937, + "train_loss": 0.49641228274062826, + "val_score": 0.603039885925274 + }, + { + "epoch": 8, + "test_score": 0.5755726662784714, + "train_loss": 0.4953731486090907, + "val_score": 0.6003639285777305 + }, + { + "epoch": 9, + "test_score": 0.5757936467446486, + "train_loss": 0.4951117623735357, + "val_score": 0.6030508239191282 + }, + { + "epoch": 10, + "test_score": 0.5747242675444127, + "train_loss": 0.49429288340939415, + "val_score": 0.6015486150266484 + }, + { + "epoch": 11, + "test_score": 0.574505664378751, + "train_loss": 0.4939393456335421, + "val_score": 0.6033995977737026 + }, + { + "epoch": 12, + "test_score": 0.5752168491704052, + "train_loss": 0.4929683832106767, + "val_score": 0.6050918211416296 + }, + { + "epoch": 13, + "test_score": 0.5763590940262089, + "train_loss": 0.49330422392597906, + "val_score": 0.6073785694362939 + }, + { + "epoch": 14, + "test_score": 0.5763749354641947, + "train_loss": 0.4932381847390422, + "val_score": 0.6080396657649799 + }, + { + "epoch": 15, + "test_score": 0.5754465980631256, + "train_loss": 0.4925268402806035, + "val_score": 0.6063322405019554 + }, + { + "epoch": 16, + "test_score": 0.5743056611875272, + "train_loss": 0.4924654452889054, + "val_score": 0.6050064472987629 + }, + { + "epoch": 17, + "test_score": 0.5731771749197139, + "train_loss": 0.4926512842928922, + "val_score": 0.6022254992143882 + }, + { + "epoch": 18, + "test_score": 0.575151363374717, + "train_loss": 0.49213752647240955, + "val_score": 0.6055324254985157 + }, + { + "epoch": 19, + "test_score": 0.5757693530269941, + "train_loss": 0.49162033752158835, + "val_score": 0.6063474645089002 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5763749354641947, + "score": 0.5763749354641947 + }, + "val": { + "roc-auc": 0.6080396657649799, + "score": 0.6080396657649799 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1", + "seed": 1, + "task_type": "binclass", + "time_seconds": 25.662135124206543, + "variant": "baseline_plr" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2.toml new file mode 100644 index 000000000..e657e755b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2.toml @@ -0,0 +1,27 @@ +dataset = "ecom-offers" +variant = "baseline_plr" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2/report.json new file mode 100644 index 000000000..b088b8366 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2/report.json @@ -0,0 +1,176 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 17, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "baseline_plr" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5647947637101525, + "train_loss": 0.5599922913092154, + "val_score": 0.5875559947525343 + }, + { + "epoch": 1, + "test_score": 0.5665267030729836, + "train_loss": 0.5211185868139621, + "val_score": 0.5884676146271168 + }, + { + "epoch": 2, + "test_score": 0.57200711533793, + "train_loss": 0.5062609061046883, + "val_score": 0.5959624216228985 + }, + { + "epoch": 3, + "test_score": 0.5730880060037655, + "train_loss": 0.5022778091607271, + "val_score": 0.5964357588845735 + }, + { + "epoch": 4, + "test_score": 0.5750484017765074, + "train_loss": 0.4995520931703073, + "val_score": 0.5986645593977806 + }, + { + "epoch": 5, + "test_score": 0.5769945833378225, + "train_loss": 0.4983021484480964, + "val_score": 0.6013675447934803 + }, + { + "epoch": 6, + "test_score": 0.5764173270406633, + "train_loss": 0.49657544272917287, + "val_score": 0.6021789390343494 + }, + { + "epoch": 7, + "test_score": 0.5755138350645392, + "train_loss": 0.4954980076463134, + "val_score": 0.6006230167791922 + }, + { + "epoch": 8, + "test_score": 0.5753642913940369, + "train_loss": 0.4957014432659856, + "val_score": 0.6007992371471607 + }, + { + "epoch": 9, + "test_score": 0.5751326177247584, + "train_loss": 0.49462021776923426, + "val_score": 0.6006985619058831 + }, + { + "epoch": 10, + "test_score": 0.574050970786008, + "train_loss": 0.4937642288428766, + "val_score": 0.5991349557837495 + }, + { + "epoch": 11, + "test_score": 0.5759820578838146, + "train_loss": 0.4934336228503121, + "val_score": 0.6043515349167372 + }, + { + "epoch": 12, + "test_score": 0.5740183425688927, + "train_loss": 0.4933987199156373, + "val_score": 0.6007254167445679 + }, + { + "epoch": 13, + "test_score": 0.5732814321002122, + "train_loss": 0.49324297849778775, + "val_score": 0.6003006149879674 + }, + { + "epoch": 14, + "test_score": 0.5745096038167632, + "train_loss": 0.49280598373324785, + "val_score": 0.6019991365470997 + }, + { + "epoch": 15, + "test_score": 0.5742709966126041, + "train_loss": 0.4923618893932413, + "val_score": 0.6030152385860228 + }, + { + "epoch": 16, + "test_score": 0.5750240956609363, + "train_loss": 0.49230552713076275, + "val_score": 0.6044452412407713 + }, + { + "epoch": 17, + "test_score": 0.57621293188747, + "train_loss": 0.49252393620985524, + "val_score": 0.6069789751857458 + }, + { + "epoch": 18, + "test_score": 0.574770899208372, + "train_loss": 0.4919488087848381, + "val_score": 0.6040690523807111 + }, + { + "epoch": 19, + "test_score": 0.5760533954961569, + "train_loss": 0.49158883149977084, + "val_score": 0.606631240588 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.57621293188747, + "score": 0.57621293188747 + }, + "val": { + "roc-auc": 0.6069789751857458, + "score": 0.6069789751857458 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2", + "seed": 2, + "task_type": "binclass", + "time_seconds": 28.923240184783936, + "variant": "baseline_plr" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..ee951091f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0.toml @@ -0,0 +1,28 @@ +dataset = "ecom-offers" +variant = "cf_fisd_only" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..ccfa17323 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,141 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 7, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "cf_fisd_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5694692246081338, + "train_loss": 0.5502510170141856, + "val_score": 0.5929137985696494 + }, + { + "epoch": 1, + "test_score": 0.5730968550167668, + "train_loss": 0.5064160205699779, + "val_score": 0.5983763572638899 + }, + { + "epoch": 2, + "test_score": 0.5747250517126401, + "train_loss": 0.5026354524824355, + "val_score": 0.6007589641024034 + }, + { + "epoch": 3, + "test_score": 0.5731190627849467, + "train_loss": 0.4991984836481236, + "val_score": 0.5989815069333099 + }, + { + "epoch": 4, + "test_score": 0.57317102865254, + "train_loss": 0.49786143722357573, + "val_score": 0.5998851849693626 + }, + { + "epoch": 5, + "test_score": 0.5740694126870087, + "train_loss": 0.49681372388645456, + "val_score": 0.6021412825582028 + }, + { + "epoch": 6, + "test_score": 0.5729867460196147, + "train_loss": 0.4954415603920265, + "val_score": 0.6008536986273423 + }, + { + "epoch": 7, + "test_score": 0.5731309058948163, + "train_loss": 0.494728273263684, + "val_score": 0.6021670244339009 + }, + { + "epoch": 8, + "test_score": 0.5719955697780599, + "train_loss": 0.494201863805453, + "val_score": 0.599877924912795 + }, + { + "epoch": 9, + "test_score": 0.5722517107358259, + "train_loss": 0.49334189185389765, + "val_score": 0.6005756126636438 + }, + { + "epoch": 10, + "test_score": 0.5715275670218046, + "train_loss": 0.4932152231534322, + "val_score": 0.5983085181160228 + }, + { + "epoch": 11, + "test_score": 0.5720281948956959, + "train_loss": 0.4935942192872365, + "val_score": 0.6002474101977806 + }, + { + "epoch": 12, + "test_score": 0.5695525943985742, + "train_loss": 0.49303652346134186, + "val_score": 0.594618124488728 + }, + { + "epoch": 13, + "test_score": 0.5718520545945235, + "train_loss": 0.49319977561632794, + "val_score": 0.5995587993602954 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5731309058948163, + "score": 0.5731309058948163 + }, + "val": { + "roc-auc": 0.6021670244339009, + "score": 0.6021670244339009 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0", + "seed": 0, + "task_type": "binclass", + "time_seconds": 18.104774713516235, + "variant": "cf_fisd_only" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1.toml new file mode 100644 index 000000000..1fad05de8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1.toml @@ -0,0 +1,28 @@ +dataset = "ecom-offers" +variant = "cf_fisd_only" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1/report.json new file mode 100644 index 000000000..51f0f9295 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1/report.json @@ -0,0 +1,135 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 6, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "cf_fisd_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5707511164974824, + "train_loss": 0.5509031525364628, + "val_score": 0.5948029721624961 + }, + { + "epoch": 1, + "test_score": 0.5710755947717638, + "train_loss": 0.5070444698686953, + "val_score": 0.5939918985434145 + }, + { + "epoch": 2, + "test_score": 0.5741403597649766, + "train_loss": 0.502915170457628, + "val_score": 0.5995196540197563 + }, + { + "epoch": 3, + "test_score": 0.5740006538413278, + "train_loss": 0.500177988299617, + "val_score": 0.6005159475287041 + }, + { + "epoch": 4, + "test_score": 0.5736018221614818, + "train_loss": 0.49774023283410956, + "val_score": 0.6005149893486494 + }, + { + "epoch": 5, + "test_score": 0.5729031065744875, + "train_loss": 0.4964937291763447, + "val_score": 0.5996165481351259 + }, + { + "epoch": 6, + "test_score": 0.5730458902809421, + "train_loss": 0.4950112937777131, + "val_score": 0.6005870260622175 + }, + { + "epoch": 7, + "test_score": 0.5718939161600558, + "train_loss": 0.49456944896115196, + "val_score": 0.5993327167764075 + }, + { + "epoch": 8, + "test_score": 0.5709125652674374, + "train_loss": 0.4939544118112988, + "val_score": 0.5975769476443196 + }, + { + "epoch": 9, + "test_score": 0.5721842846661835, + "train_loss": 0.49402084008411123, + "val_score": 0.6001584616062485 + }, + { + "epoch": 10, + "test_score": 0.571063903536373, + "train_loss": 0.493623247301137, + "val_score": 0.5976913875106123 + }, + { + "epoch": 11, + "test_score": 0.5712434129713921, + "train_loss": 0.49344748148211726, + "val_score": 0.5978700070140254 + }, + { + "epoch": 12, + "test_score": 0.5726795101478993, + "train_loss": 0.49258249280629335, + "val_score": 0.6005388001230066 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5730458902809421, + "score": 0.5730458902809421 + }, + "val": { + "roc-auc": 0.6005870260622175, + "score": 0.6005870260622175 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1", + "seed": 1, + "task_type": "binclass", + "time_seconds": 16.94504404067993, + "variant": "cf_fisd_only" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2.toml new file mode 100644 index 000000000..42bdd942d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2.toml @@ -0,0 +1,28 @@ +dataset = "ecom-offers" +variant = "cf_fisd_only" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2/report.json new file mode 100644 index 000000000..75d607f7d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2/report.json @@ -0,0 +1,135 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 6, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "cf_fisd_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5664456392950381, + "train_loss": 0.5500196737271769, + "val_score": 0.5873379829929527 + }, + { + "epoch": 1, + "test_score": 0.5735965127536811, + "train_loss": 0.5067310901703658, + "val_score": 0.599813325150574 + }, + { + "epoch": 2, + "test_score": 0.5743384784728712, + "train_loss": 0.5025817271735933, + "val_score": 0.60041159435014 + }, + { + "epoch": 3, + "test_score": 0.5735590834433473, + "train_loss": 0.4996792955531014, + "val_score": 0.598594505379864 + }, + { + "epoch": 4, + "test_score": 0.5730344687002382, + "train_loss": 0.4973104816895944, + "val_score": 0.598631505871204 + }, + { + "epoch": 5, + "test_score": 0.5734397875899657, + "train_loss": 0.49601822153285696, + "val_score": 0.5994892354883299 + }, + { + "epoch": 6, + "test_score": 0.5737641046913308, + "train_loss": 0.49453101831453816, + "val_score": 0.6010747507659433 + }, + { + "epoch": 7, + "test_score": 0.5717324239973925, + "train_loss": 0.4938915134580047, + "val_score": 0.5972482918855844 + }, + { + "epoch": 8, + "test_score": 0.5721690042339257, + "train_loss": 0.49456112141962405, + "val_score": 0.5987011839877145 + }, + { + "epoch": 9, + "test_score": 0.5718806255934186, + "train_loss": 0.4939179475660677, + "val_score": 0.5980858628684842 + }, + { + "epoch": 10, + "test_score": 0.5710394114520523, + "train_loss": 0.4932858695586522, + "val_score": 0.5962455085494205 + }, + { + "epoch": 11, + "test_score": 0.5725301958388551, + "train_loss": 0.4931839207808177, + "val_score": 0.5989351936689005 + }, + { + "epoch": 12, + "test_score": 0.5714935905312548, + "train_loss": 0.4932064943843418, + "val_score": 0.5970939990996351 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5737641046913308, + "score": 0.5737641046913308 + }, + "val": { + "roc-auc": 0.6010747507659433, + "score": 0.6010747507659433 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2", + "seed": 2, + "task_type": "binclass", + "time_seconds": 18.815613985061646, + "variant": "cf_fisd_only" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0.toml new file mode 100644 index 000000000..65294a334 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0.toml @@ -0,0 +1,29 @@ +dataset = "ecom-offers" +variant = "esam_only" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0/report.json new file mode 100644 index 000000000..09447b34f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 14, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "esam_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5751306650528877, + "train_loss": 0.56087240907881, + "val_score": 0.6017959286693619 + }, + { + "epoch": 1, + "test_score": 0.5677375920937394, + "train_loss": 0.5205924102553615, + "val_score": 0.5893268183674035 + }, + { + "epoch": 2, + "test_score": 0.5716916720453995, + "train_loss": 0.5064562239028789, + "val_score": 0.5955753389926788 + }, + { + "epoch": 3, + "test_score": 0.5727886149138035, + "train_loss": 0.501467732919587, + "val_score": 0.5971922457230048 + }, + { + "epoch": 4, + "test_score": 0.574375461458206, + "train_loss": 0.49999182643713774, + "val_score": 0.5995666085277406 + }, + { + "epoch": 5, + "test_score": 0.5756433963928981, + "train_loss": 0.49881415676187585, + "val_score": 0.6016288699768387 + }, + { + "epoch": 6, + "test_score": 0.5759478210370156, + "train_loss": 0.4973221161851176, + "val_score": 0.6020441488978194 + }, + { + "epoch": 7, + "test_score": 0.577360008831284, + "train_loss": 0.4961576544576221, + "val_score": 0.6040952438639735 + }, + { + "epoch": 8, + "test_score": 0.575118840539893, + "train_loss": 0.4953662283994533, + "val_score": 0.6016707719276889 + }, + { + "epoch": 9, + "test_score": 0.5755175668374476, + "train_loss": 0.4941921719798335, + "val_score": 0.603020066339375 + }, + { + "epoch": 10, + "test_score": 0.5742576099621128, + "train_loss": 0.49376690994810174, + "val_score": 0.6027129143522503 + }, + { + "epoch": 11, + "test_score": 0.5748242226478373, + "train_loss": 0.4941109347122687, + "val_score": 0.6034782422443395 + }, + { + "epoch": 12, + "test_score": 0.5729161336853962, + "train_loss": 0.49326626901273374, + "val_score": 0.6008168418630104 + }, + { + "epoch": 13, + "test_score": 0.5751381038028711, + "train_loss": 0.49349481971175585, + "val_score": 0.6060124773911888 + }, + { + "epoch": 14, + "test_score": 0.5766903074687046, + "train_loss": 0.4929675846188157, + "val_score": 0.6082039420500369 + }, + { + "epoch": 15, + "test_score": 0.5762027593968676, + "train_loss": 0.4927098403374354, + "val_score": 0.6080009331789258 + }, + { + "epoch": 16, + "test_score": 0.5763376766252147, + "train_loss": 0.4920907649728987, + "val_score": 0.6077780973513 + }, + { + "epoch": 17, + "test_score": 0.5756898017948786, + "train_loss": 0.49206242130862343, + "val_score": 0.6070591195766991 + }, + { + "epoch": 18, + "test_score": 0.5761874479698181, + "train_loss": 0.4918863221451088, + "val_score": 0.6075633802569831 + }, + { + "epoch": 19, + "test_score": 0.575851625601812, + "train_loss": 0.49199968190104876, + "val_score": 0.6074322643724318 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5766903074687046, + "score": 0.5766903074687046 + }, + "val": { + "roc-auc": 0.6082039420500369, + "score": 0.6082039420500369 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0", + "seed": 0, + "task_type": "binclass", + "time_seconds": 33.07249736785889, + "variant": "esam_only" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1.toml new file mode 100644 index 000000000..e2261c903 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1.toml @@ -0,0 +1,29 @@ +dataset = "ecom-offers" +variant = "esam_only" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1/report.json new file mode 100644 index 000000000..f2ab418cc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1/report.json @@ -0,0 +1,178 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 14, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "esam_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5746909419444866, + "train_loss": 0.5611117923701251, + "val_score": 0.6008023401764144 + }, + { + "epoch": 1, + "test_score": 0.5635703609602598, + "train_loss": 0.5212651198660886, + "val_score": 0.58227185655508 + }, + { + "epoch": 2, + "test_score": 0.5711950301017077, + "train_loss": 0.5066843419163315, + "val_score": 0.5939066757981717 + }, + { + "epoch": 3, + "test_score": 0.5726653734234515, + "train_loss": 0.5021584519633541, + "val_score": 0.5962150089412203 + }, + { + "epoch": 4, + "test_score": 0.5744955569772109, + "train_loss": 0.4999331335226695, + "val_score": 0.5984644729758362 + }, + { + "epoch": 5, + "test_score": 0.5756790187068817, + "train_loss": 0.4988091301034998, + "val_score": 0.6007560158560814 + }, + { + "epoch": 6, + "test_score": 0.5770970552183247, + "train_loss": 0.4973242293905329, + "val_score": 0.6032151444278802 + }, + { + "epoch": 7, + "test_score": 0.5772226554123173, + "train_loss": 0.4965477806550485, + "val_score": 0.6029452029946458 + }, + { + "epoch": 8, + "test_score": 0.5755433328077031, + "train_loss": 0.49546305117783723, + "val_score": 0.6004737360119901 + }, + { + "epoch": 9, + "test_score": 0.5763837038907378, + "train_loss": 0.4952508856852849, + "val_score": 0.6034021959157737 + }, + { + "epoch": 10, + "test_score": 0.5750985296529589, + "train_loss": 0.4944774690601561, + "val_score": 0.6018824081045995 + }, + { + "epoch": 11, + "test_score": 0.5751528976169009, + "train_loss": 0.4940486892505928, + "val_score": 0.6037345369824112 + }, + { + "epoch": 12, + "test_score": 0.5758723146252036, + "train_loss": 0.49305023142585047, + "val_score": 0.6053632403833327 + }, + { + "epoch": 13, + "test_score": 0.5768479128845014, + "train_loss": 0.493441888027721, + "val_score": 0.6077879002703204 + }, + { + "epoch": 14, + "test_score": 0.5767053647384629, + "train_loss": 0.4933244977836256, + "val_score": 0.6084306806187314 + }, + { + "epoch": 15, + "test_score": 0.5760965774398111, + "train_loss": 0.4926842865016725, + "val_score": 0.607144147000623 + }, + { + "epoch": 16, + "test_score": 0.5745206410620511, + "train_loss": 0.49252080696600453, + "val_score": 0.6051915308322366 + }, + { + "epoch": 17, + "test_score": 0.5735734092360238, + "train_loss": 0.4926967184852671, + "val_score": 0.6027158662838801 + }, + { + "epoch": 18, + "test_score": 0.5757706083160539, + "train_loss": 0.49223133700865285, + "val_score": 0.6061065006517025 + }, + { + "epoch": 19, + "test_score": 0.5757061701443216, + "train_loss": 0.49164940140865465, + "val_score": 0.6061247797788983 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5767053647384629, + "score": 0.5767053647384629 + }, + "val": { + "roc-auc": 0.6084306806187314, + "score": 0.6084306806187314 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1", + "seed": 1, + "task_type": "binclass", + "time_seconds": 32.38373279571533, + "variant": "esam_only" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2.toml new file mode 100644 index 000000000..5b83af7db --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2.toml @@ -0,0 +1,29 @@ +dataset = "ecom-offers" +variant = "esam_only" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2/report.json new file mode 100644 index 000000000..00e25e38e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2/report.json @@ -0,0 +1,178 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 19, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "esam_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5646824633812344, + "train_loss": 0.5600938432746463, + "val_score": 0.5874748921815265 + }, + { + "epoch": 1, + "test_score": 0.5663432046082858, + "train_loss": 0.5211920037313744, + "val_score": 0.5882059172276601 + }, + { + "epoch": 2, + "test_score": 0.5718646601762665, + "train_loss": 0.5063478036059273, + "val_score": 0.5957459724338674 + }, + { + "epoch": 3, + "test_score": 0.5730667714720417, + "train_loss": 0.5023864270360382, + "val_score": 0.5964439034150377 + }, + { + "epoch": 4, + "test_score": 0.5750929908837003, + "train_loss": 0.49960060472841616, + "val_score": 0.5986022408411512 + }, + { + "epoch": 5, + "test_score": 0.5769539925587457, + "train_loss": 0.49839139646954006, + "val_score": 0.6012458485559276 + }, + { + "epoch": 6, + "test_score": 0.5764294336062614, + "train_loss": 0.49663131766849095, + "val_score": 0.6021241458764567 + }, + { + "epoch": 7, + "test_score": 0.5755681100441066, + "train_loss": 0.4956015679571364, + "val_score": 0.6006771318404308 + }, + { + "epoch": 8, + "test_score": 0.5754694102297417, + "train_loss": 0.49574901163578033, + "val_score": 0.6008344318376285 + }, + { + "epoch": 9, + "test_score": 0.5752777756322495, + "train_loss": 0.49462479960035394, + "val_score": 0.6007017275853714 + }, + { + "epoch": 10, + "test_score": 0.5740121312126565, + "train_loss": 0.49387454986572266, + "val_score": 0.5990072156412368 + }, + { + "epoch": 11, + "test_score": 0.5758652261163651, + "train_loss": 0.4934555744683301, + "val_score": 0.6041306854700706 + }, + { + "epoch": 12, + "test_score": 0.5736422331707673, + "train_loss": 0.49341002437803483, + "val_score": 0.600174452157237 + }, + { + "epoch": 13, + "test_score": 0.5734670599070928, + "train_loss": 0.4933059408708855, + "val_score": 0.6011017567022521 + }, + { + "epoch": 14, + "test_score": 0.5745859718837815, + "train_loss": 0.4929176900121901, + "val_score": 0.6021131857707549 + }, + { + "epoch": 15, + "test_score": 0.5744547430356346, + "train_loss": 0.49251379293424113, + "val_score": 0.6033664152613495 + }, + { + "epoch": 16, + "test_score": 0.5753802196174392, + "train_loss": 0.4923371247671269, + "val_score": 0.6050771020218675 + }, + { + "epoch": 17, + "test_score": 0.5764116147005718, + "train_loss": 0.4924943822401541, + "val_score": 0.607187338809239 + }, + { + "epoch": 18, + "test_score": 0.5751273269138326, + "train_loss": 0.49204753670427537, + "val_score": 0.6045122806768237 + }, + { + "epoch": 19, + "test_score": 0.5764039992802762, + "train_loss": 0.4917129118133474, + "val_score": 0.6073509185711023 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5764039992802762, + "score": 0.5764039992802762 + }, + "val": { + "roc-auc": 0.6073509185711023, + "score": 0.6073509185711023 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2", + "seed": 2, + "task_type": "binclass", + "time_seconds": 36.74677538871765, + "variant": "esam_only" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0.toml new file mode 100644 index 000000000..ed6cb5c3a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0.toml @@ -0,0 +1,29 @@ +dataset = "ecom-offers" +variant = "mfb_only" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..163bc3268 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0/report.json @@ -0,0 +1,100 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "mfb_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5756126061669247, + "train_loss": 0.5612001937848551, + "val_score": 0.6017022260306357 + }, + { + "epoch": 1, + "test_score": 0.5678979560459885, + "train_loss": 0.5231215937270058, + "val_score": 0.5896429772470416 + }, + { + "epoch": 2, + "test_score": 0.5704295176393404, + "train_loss": 0.5125225379511162, + "val_score": 0.5927430766810713 + }, + { + "epoch": 3, + "test_score": 0.5706981309012455, + "train_loss": 0.508107524227213, + "val_score": 0.5932826057579811 + }, + { + "epoch": 4, + "test_score": 0.5717370918130072, + "train_loss": 0.5076041861816689, + "val_score": 0.5947484922557751 + }, + { + "epoch": 5, + "test_score": 0.5731379541104258, + "train_loss": 0.5074608066567668, + "val_score": 0.5966676052900207 + }, + { + "epoch": 6, + "test_score": 0.5749101960007718, + "train_loss": 0.5042634120693913, + "val_score": 0.5994873007016811 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5756126061669247, + "score": 0.5756126061669247 + }, + "val": { + "roc-auc": 0.6017022260306357, + "score": 0.6017022260306357 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0", + "seed": 0, + "task_type": "binclass", + "time_seconds": 9.245736360549927, + "variant": "mfb_only" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1.toml new file mode 100644 index 000000000..dc012a660 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1.toml @@ -0,0 +1,29 @@ +dataset = "ecom-offers" +variant = "mfb_only" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1/report.json new file mode 100644 index 000000000..d424359d0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1/report.json @@ -0,0 +1,100 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "mfb_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5758805406428937, + "train_loss": 0.5612830320994059, + "val_score": 0.6019166519856285 + }, + { + "epoch": 1, + "test_score": 0.5639916173718269, + "train_loss": 0.5240427851676941, + "val_score": 0.5835630336611454 + }, + { + "epoch": 2, + "test_score": 0.5662793181438439, + "train_loss": 0.5127950227922864, + "val_score": 0.5855571279699732 + }, + { + "epoch": 3, + "test_score": 0.5687952242679598, + "train_loss": 0.5094026978369113, + "val_score": 0.5900073252128114 + }, + { + "epoch": 4, + "test_score": 0.5723007940878004, + "train_loss": 0.5075066724309215, + "val_score": 0.5948114889090584 + }, + { + "epoch": 5, + "test_score": 0.5723376406960523, + "train_loss": 0.5056416751058014, + "val_score": 0.5956702614683207 + }, + { + "epoch": 6, + "test_score": 0.5747176408579691, + "train_loss": 0.5049899849626753, + "val_score": 0.5981938313341004 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5758805406428937, + "score": 0.5758805406428937 + }, + "val": { + "roc-auc": 0.6019166519856285, + "score": 0.6019166519856285 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1", + "seed": 1, + "task_type": "binclass", + "time_seconds": 9.183435678482056, + "variant": "mfb_only" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2.toml new file mode 100644 index 000000000..976fb7e3c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2.toml @@ -0,0 +1,29 @@ +dataset = "ecom-offers" +variant = "mfb_only" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2/report.json new file mode 100644 index 000000000..ae147922f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2/report.json @@ -0,0 +1,178 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 17, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "mfb_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5655640513381363, + "train_loss": 0.5601579028147238, + "val_score": 0.5890461121497865 + }, + { + "epoch": 1, + "test_score": 0.566298851061509, + "train_loss": 0.5232984291182624, + "val_score": 0.5876600125680793 + }, + { + "epoch": 2, + "test_score": 0.565342370389669, + "train_loss": 0.5126102323885318, + "val_score": 0.5864489282586565 + }, + { + "epoch": 3, + "test_score": 0.56817462485423, + "train_loss": 0.5099104096492132, + "val_score": 0.5899897942031197 + }, + { + "epoch": 4, + "test_score": 0.5717861255733153, + "train_loss": 0.5082687989429191, + "val_score": 0.5945968860592866 + }, + { + "epoch": 5, + "test_score": 0.5718616877757521, + "train_loss": 0.5065136921626551, + "val_score": 0.5957169395782125 + }, + { + "epoch": 6, + "test_score": 0.5747729355661799, + "train_loss": 0.5042106216704404, + "val_score": 0.6006416017869438 + }, + { + "epoch": 7, + "test_score": 0.5739895267111441, + "train_loss": 0.5042047484053506, + "val_score": 0.5990293459151907 + }, + { + "epoch": 8, + "test_score": 0.5723069682502868, + "train_loss": 0.5039668894476361, + "val_score": 0.5963988800083944 + }, + { + "epoch": 9, + "test_score": 0.5764932115889327, + "train_loss": 0.503834640538251, + "val_score": 0.6029036769452016 + }, + { + "epoch": 10, + "test_score": 0.5764806679967731, + "train_loss": 0.499090200221097, + "val_score": 0.6023098264298107 + }, + { + "epoch": 11, + "test_score": 0.5769842279779496, + "train_loss": 0.4986710399389267, + "val_score": 0.6036243315348981 + }, + { + "epoch": 12, + "test_score": 0.5764756344426175, + "train_loss": 0.5011906392044492, + "val_score": 0.6016404639554996 + }, + { + "epoch": 13, + "test_score": 0.5759788406244468, + "train_loss": 0.5009944047088977, + "val_score": 0.6021128946314307 + }, + { + "epoch": 14, + "test_score": 0.5777112418096728, + "train_loss": 0.49901655702679243, + "val_score": 0.6042899349951487 + }, + { + "epoch": 15, + "test_score": 0.5778695632051555, + "train_loss": 0.4992442086890892, + "val_score": 0.6056838105765301 + }, + { + "epoch": 16, + "test_score": 0.5785535717635366, + "train_loss": 0.5002761604609313, + "val_score": 0.6056769964422184 + }, + { + "epoch": 17, + "test_score": 0.5816249076525681, + "train_loss": 0.4998697401196868, + "val_score": 0.6106538647226952 + }, + { + "epoch": 18, + "test_score": 0.580032497543074, + "train_loss": 0.498333211850237, + "val_score": 0.6061102338686076 + }, + { + "epoch": 19, + "test_score": 0.5802514540493597, + "train_loss": 0.498783348887055, + "val_score": 0.6063259275695186 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5816249076525681, + "score": 0.5816249076525681 + }, + "val": { + "roc-auc": 0.6106538647226952, + "score": 0.6106538647226952 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2", + "seed": 2, + "task_type": "binclass", + "time_seconds": 28.520503282546997, + "variant": "mfb_only" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0.toml new file mode 100644 index 000000000..27ba0bcca --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0.toml @@ -0,0 +1,30 @@ +dataset = "ecom-offers" +variant = "rla_only" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0/report.json new file mode 100644 index 000000000..f53f8f9e8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 19, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "rla_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5738058639740508, + "train_loss": 0.5600903784787213, + "val_score": 0.5996591097560906 + }, + { + "epoch": 1, + "test_score": 0.566052271996951, + "train_loss": 0.5203855821379909, + "val_score": 0.5867063728127937 + }, + { + "epoch": 2, + "test_score": 0.5697889358834675, + "train_loss": 0.5058877644715486, + "val_score": 0.5926538516914465 + }, + { + "epoch": 3, + "test_score": 0.5724541067249609, + "train_loss": 0.5015750681912458, + "val_score": 0.5966838501272544 + }, + { + "epoch": 4, + "test_score": 0.5748546719311031, + "train_loss": 0.4996969308014269, + "val_score": 0.6004681196027468 + }, + { + "epoch": 5, + "test_score": 0.5757011892813118, + "train_loss": 0.4983945345437085, + "val_score": 0.6016276869930021 + }, + { + "epoch": 6, + "test_score": 0.5777511104101054, + "train_loss": 0.49719943823637786, + "val_score": 0.6039715465042297 + }, + { + "epoch": 7, + "test_score": 0.5767656991997876, + "train_loss": 0.4959517259288717, + "val_score": 0.603024860924956 + }, + { + "epoch": 8, + "test_score": 0.576220767370786, + "train_loss": 0.4950623898594468, + "val_score": 0.6032534237210622 + }, + { + "epoch": 9, + "test_score": 0.5740754287760579, + "train_loss": 0.4947311133146286, + "val_score": 0.6016036108764757 + }, + { + "epoch": 10, + "test_score": 0.5750780606925876, + "train_loss": 0.4943293062625108, + "val_score": 0.60343813135313 + }, + { + "epoch": 11, + "test_score": 0.5747031631908137, + "train_loss": 0.4936874199796606, + "val_score": 0.6036642581607127 + }, + { + "epoch": 12, + "test_score": 0.5746027710608276, + "train_loss": 0.49342205844543596, + "val_score": 0.6047218457106929 + }, + { + "epoch": 13, + "test_score": 0.5753453008852248, + "train_loss": 0.49321615254437484, + "val_score": 0.6057724164344271 + }, + { + "epoch": 14, + "test_score": 0.5750546534259727, + "train_loss": 0.49275959255518736, + "val_score": 0.6064825457847545 + }, + { + "epoch": 15, + "test_score": 0.575529735392629, + "train_loss": 0.4924712076231285, + "val_score": 0.6070732601031206 + }, + { + "epoch": 16, + "test_score": 0.5750690164123993, + "train_loss": 0.492143577447644, + "val_score": 0.6063755060167295 + }, + { + "epoch": 17, + "test_score": 0.5759319579026756, + "train_loss": 0.4919159158512398, + "val_score": 0.6079652409718912 + }, + { + "epoch": 18, + "test_score": 0.5761580649073832, + "train_loss": 0.4918079514194418, + "val_score": 0.6090219661598224 + }, + { + "epoch": 19, + "test_score": 0.5766629173713693, + "train_loss": 0.49131932689083946, + "val_score": 0.6098132349342364 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5766629173713693, + "score": 0.5766629173713693 + }, + "val": { + "roc-auc": 0.6098132349342364, + "score": 0.6098132349342364 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0", + "seed": 0, + "task_type": "binclass", + "time_seconds": 27.724525213241577, + "variant": "rla_only" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1.toml new file mode 100644 index 000000000..c2afea846 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1.toml @@ -0,0 +1,30 @@ +dataset = "ecom-offers" +variant = "rla_only" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1/report.json new file mode 100644 index 000000000..f477b0fc3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1/report.json @@ -0,0 +1,179 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 17, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "rla_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5744099679596921, + "train_loss": 0.5611426466041141, + "val_score": 0.6003674480467773 + }, + { + "epoch": 1, + "test_score": 0.5661563215123454, + "train_loss": 0.5209181766819071, + "val_score": 0.5867286357578321 + }, + { + "epoch": 2, + "test_score": 0.5714187226121498, + "train_loss": 0.5059594881755335, + "val_score": 0.5942213052750304 + }, + { + "epoch": 3, + "test_score": 0.5729606174102976, + "train_loss": 0.5019116169876523, + "val_score": 0.5964863655326891 + }, + { + "epoch": 4, + "test_score": 0.5735315166757, + "train_loss": 0.5001836750242445, + "val_score": 0.5971035735295656 + }, + { + "epoch": 5, + "test_score": 0.574901839804957, + "train_loss": 0.4984646196718569, + "val_score": 0.5994008654901384 + }, + { + "epoch": 6, + "test_score": 0.5767816801143355, + "train_loss": 0.4975106026287432, + "val_score": 0.6025049192595473 + }, + { + "epoch": 7, + "test_score": 0.5780280767683461, + "train_loss": 0.49668018243930956, + "val_score": 0.604435957950165 + }, + { + "epoch": 8, + "test_score": 0.5764066989266243, + "train_loss": 0.4956071581001635, + "val_score": 0.6016519326336919 + }, + { + "epoch": 9, + "test_score": 0.5754726646828595, + "train_loss": 0.4948139527329692, + "val_score": 0.6022930730200864 + }, + { + "epoch": 10, + "test_score": 0.5756381861684305, + "train_loss": 0.4942547689985346, + "val_score": 0.6035153864626877 + }, + { + "epoch": 11, + "test_score": 0.5749532849600514, + "train_loss": 0.49325044563523046, + "val_score": 0.6039946275876223 + }, + { + "epoch": 12, + "test_score": 0.574489857035036, + "train_loss": 0.493444146933379, + "val_score": 0.6038499866230694 + }, + { + "epoch": 13, + "test_score": 0.5753126292754014, + "train_loss": 0.4927494956387414, + "val_score": 0.6059532028988868 + }, + { + "epoch": 14, + "test_score": 0.5750983715795217, + "train_loss": 0.49215994499347826, + "val_score": 0.6051037983923124 + }, + { + "epoch": 15, + "test_score": 0.5755156389614102, + "train_loss": 0.492595124023932, + "val_score": 0.6076091812635939 + }, + { + "epoch": 16, + "test_score": 0.5749148452195115, + "train_loss": 0.49249838458167183, + "val_score": 0.6060914977632319 + }, + { + "epoch": 17, + "test_score": 0.5767242343675878, + "train_loss": 0.49214974487269364, + "val_score": 0.6088361897884629 + }, + { + "epoch": 18, + "test_score": 0.5751086773477281, + "train_loss": 0.4915305613367646, + "val_score": 0.6059502251701017 + }, + { + "epoch": 19, + "test_score": 0.5759139437297989, + "train_loss": 0.4917481326394611, + "val_score": 0.6069593914595525 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5767242343675878, + "score": 0.5767242343675878 + }, + "val": { + "roc-auc": 0.6088361897884629, + "score": 0.6088361897884629 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1", + "seed": 1, + "task_type": "binclass", + "time_seconds": 28.521148443222046, + "variant": "rla_only" +} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2.toml new file mode 100644 index 000000000..983c8f37c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2.toml @@ -0,0 +1,30 @@ +dataset = "ecom-offers" +variant = "rla_only" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/ecom-offers" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2/report.json new file mode 100644 index 000000000..dbef5ef90 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2/report.json @@ -0,0 +1,179 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 19, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/ecom-offers" + }, + "dataset": "ecom-offers", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "rla_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2.toml", + "dataset": "ecom-offers", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.5682818730321174, + "train_loss": 0.5605212438989569, + "val_score": 0.5927119874236068 + }, + { + "epoch": 1, + "test_score": 0.5672713870353969, + "train_loss": 0.5217861185471216, + "val_score": 0.5892199739206975 + }, + { + "epoch": 2, + "test_score": 0.571274286883303, + "train_loss": 0.506144740515285, + "val_score": 0.5945899687363537 + }, + { + "epoch": 3, + "test_score": 0.5740664526844113, + "train_loss": 0.5019352408470931, + "val_score": 0.5979948099661399 + }, + { + "epoch": 4, + "test_score": 0.5757866233248725, + "train_loss": 0.4998747734008012, + "val_score": 0.6000116979043437 + }, + { + "epoch": 5, + "test_score": 0.5751457347205626, + "train_loss": 0.49779329476533113, + "val_score": 0.5992338436506943 + }, + { + "epoch": 6, + "test_score": 0.5762034660781161, + "train_loss": 0.4968773893736027, + "val_score": 0.6016685054633288 + }, + { + "epoch": 7, + "test_score": 0.5753777493325488, + "train_loss": 0.49569592983634386, + "val_score": 0.6005299443281173 + }, + { + "epoch": 8, + "test_score": 0.5747554111110105, + "train_loss": 0.49488269951608443, + "val_score": 0.5998391591589697 + }, + { + "epoch": 9, + "test_score": 0.5754841451536674, + "train_loss": 0.49462440113226575, + "val_score": 0.6011704324150131 + }, + { + "epoch": 10, + "test_score": 0.5759761843708069, + "train_loss": 0.4938305141749205, + "val_score": 0.6034632651530242 + }, + { + "epoch": 11, + "test_score": 0.5751711783449855, + "train_loss": 0.49337507177282264, + "val_score": 0.6030761346138018 + }, + { + "epoch": 12, + "test_score": 0.573761587914253, + "train_loss": 0.49311621652709114, + "val_score": 0.6010109138624582 + }, + { + "epoch": 13, + "test_score": 0.5745490849823007, + "train_loss": 0.4942300303114785, + "val_score": 0.6023989740279694 + }, + { + "epoch": 14, + "test_score": 0.5733634908109679, + "train_loss": 0.49301944121166513, + "val_score": 0.6001994532860468 + }, + { + "epoch": 15, + "test_score": 0.5757922860732975, + "train_loss": 0.49234316856772814, + "val_score": 0.6060068683525613 + }, + { + "epoch": 16, + "test_score": 0.5758632796434527, + "train_loss": 0.492086089319653, + "val_score": 0.6062278062466175 + }, + { + "epoch": 17, + "test_score": 0.5762892038706345, + "train_loss": 0.49177511422722425, + "val_score": 0.6073894079268349 + }, + { + "epoch": 18, + "test_score": 0.5743713763446734, + "train_loss": 0.49189673750488844, + "val_score": 0.6031209626991266 + }, + { + "epoch": 19, + "test_score": 0.5770335251939862, + "train_loss": 0.4917442583375507, + "val_score": 0.6081233738486749 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.5770335251939862, + "score": 0.5770335251939862 + }, + "val": { + "roc-auc": 0.6081233738486749, + "score": 0.6081233738486749 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2", + "seed": 2, + "task_type": "binclass", + "time_seconds": 30.517080783843994, + "variant": "rla_only" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..75c67868d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0.toml @@ -0,0 +1,35 @@ +dataset = "homesite-insurance" +variant = "all_four_combined" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..b1fd177fb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0/report.json @@ -0,0 +1,184 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 19, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_four_combined" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.9281404176737119, + "train_loss": 0.3755828700282357, + "val_score": 0.9251322733226822 + }, + { + "epoch": 1, + "test_score": 0.9402497021027693, + "train_loss": 0.25816836140372534, + "val_score": 0.936835678138968 + }, + { + "epoch": 2, + "test_score": 0.9400969237598616, + "train_loss": 0.25369841456413267, + "val_score": 0.9375367433223097 + }, + { + "epoch": 3, + "test_score": 0.9444803308575334, + "train_loss": 0.24889986542138187, + "val_score": 0.9411922560440782 + }, + { + "epoch": 4, + "test_score": 0.9431998850066914, + "train_loss": 0.24193649928678165, + "val_score": 0.9412456241453959 + }, + { + "epoch": 5, + "test_score": 0.9439822004894514, + "train_loss": 0.23912576938217336, + "val_score": 0.9424856456072856 + }, + { + "epoch": 6, + "test_score": 0.9431244381656779, + "train_loss": 0.23391161195256494, + "val_score": 0.9426291592016532 + }, + { + "epoch": 7, + "test_score": 0.9442989443289539, + "train_loss": 0.22139542956243863, + "val_score": 0.9440450404267323 + }, + { + "epoch": 8, + "test_score": 0.9454798172632811, + "train_loss": 0.21870070519772442, + "val_score": 0.9446249591408717 + }, + { + "epoch": 9, + "test_score": 0.9452781223055992, + "train_loss": 0.21202451532537286, + "val_score": 0.9450447383826217 + }, + { + "epoch": 10, + "test_score": 0.9455226932983104, + "train_loss": 0.21138894761150534, + "val_score": 0.9456571615235437 + }, + { + "epoch": 11, + "test_score": 0.9470071536894556, + "train_loss": 0.20695550563660534, + "val_score": 0.9463883748484702 + }, + { + "epoch": 12, + "test_score": 0.9496032498898924, + "train_loss": 0.20267397273670543, + "val_score": 0.9468419597491273 + }, + { + "epoch": 13, + "test_score": 0.9479806172776625, + "train_loss": 0.20230170339345932, + "val_score": 0.9473267214681163 + }, + { + "epoch": 14, + "test_score": 0.946599535368271, + "train_loss": 0.19895012825727462, + "val_score": 0.9474537674424232 + }, + { + "epoch": 15, + "test_score": 0.9487991279142535, + "train_loss": 0.19738293412056837, + "val_score": 0.9477246874875647 + }, + { + "epoch": 16, + "test_score": 0.9474314923353743, + "train_loss": 0.19684043594382025, + "val_score": 0.9485214284004848 + }, + { + "epoch": 17, + "test_score": 0.9470441220374943, + "train_loss": 0.1940444149754264, + "val_score": 0.9475463043891069 + }, + { + "epoch": 18, + "test_score": 0.9491013260377306, + "train_loss": 0.19452549476515163, + "val_score": 0.9486381436467773 + }, + { + "epoch": 19, + "test_score": 0.9488481290971439, + "train_loss": 0.19246512664989993, + "val_score": 0.948915692942066 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9488481290971439, + "score": 0.9488481290971439 + }, + "val": { + "roc-auc": 0.948915692942066, + "score": 0.948915692942066 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0", + "seed": 0, + "task_type": "binclass", + "time_seconds": 93.77295994758606, + "variant": "all_four_combined" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1.toml new file mode 100644 index 000000000..ca8babc0b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1.toml @@ -0,0 +1,35 @@ +dataset = "homesite-insurance" +variant = "all_four_combined" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1/report.json new file mode 100644 index 000000000..eef49da80 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1/report.json @@ -0,0 +1,184 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 16, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_four_combined" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.9288299015269522, + "train_loss": 0.37169375148686495, + "val_score": 0.9242781990673085 + }, + { + "epoch": 1, + "test_score": 0.9391231943861461, + "train_loss": 0.26537797302007676, + "val_score": 0.9354944858794071 + }, + { + "epoch": 2, + "test_score": 0.9417845409289871, + "train_loss": 0.25233914811502806, + "val_score": 0.9386876479678965 + }, + { + "epoch": 3, + "test_score": 0.9425111743476582, + "train_loss": 0.24839151420376518, + "val_score": 0.9399943663682699 + }, + { + "epoch": 4, + "test_score": 0.9441030483278274, + "train_loss": 0.23347439156337219, + "val_score": 0.9413839416033751 + }, + { + "epoch": 5, + "test_score": 0.9455693990608975, + "train_loss": 0.23166601644320922, + "val_score": 0.9424608342758328 + }, + { + "epoch": 6, + "test_score": 0.9445222283186436, + "train_loss": 0.23105967830527913, + "val_score": 0.9430947277468074 + }, + { + "epoch": 7, + "test_score": 0.9461559755976042, + "train_loss": 0.2228196923028339, + "val_score": 0.9438730667756982 + }, + { + "epoch": 8, + "test_score": 0.9464112867411015, + "train_loss": 0.21721381694078445, + "val_score": 0.9451223990809644 + }, + { + "epoch": 9, + "test_score": 0.946084950460964, + "train_loss": 0.21338909674774517, + "val_score": 0.9455819538235977 + }, + { + "epoch": 10, + "test_score": 0.9466914729919881, + "train_loss": 0.21197187114845623, + "val_score": 0.9456509279182992 + }, + { + "epoch": 11, + "test_score": 0.9465127443184965, + "train_loss": 0.20536432604898106, + "val_score": 0.9458262601549519 + }, + { + "epoch": 12, + "test_score": 0.945089656217559, + "train_loss": 0.20558685606176202, + "val_score": 0.9459539479532121 + }, + { + "epoch": 13, + "test_score": 0.9471234106872256, + "train_loss": 0.20297221622683786, + "val_score": 0.9451995410448792 + }, + { + "epoch": 14, + "test_score": 0.9479524560948916, + "train_loss": 0.2004195828329433, + "val_score": 0.9473089526159311 + }, + { + "epoch": 15, + "test_score": 0.9499482515614414, + "train_loss": 0.19796687323938716, + "val_score": 0.9473182986277434 + }, + { + "epoch": 16, + "test_score": 0.9496361106437045, + "train_loss": 0.19757200046019122, + "val_score": 0.9483863024780541 + }, + { + "epoch": 17, + "test_score": 0.9472061183050925, + "train_loss": 0.19716880091212013, + "val_score": 0.9478469505546034 + }, + { + "epoch": 18, + "test_score": 0.9489066502337125, + "train_loss": 0.19436502781781284, + "val_score": 0.9483601987065858 + }, + { + "epoch": 19, + "test_score": 0.9500448404315621, + "train_loss": 0.19395294690674, + "val_score": 0.9482018792007486 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9496361106437045, + "score": 0.9496361106437045 + }, + "val": { + "roc-auc": 0.9483863024780541, + "score": 0.9483863024780541 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1", + "seed": 1, + "task_type": "binclass", + "time_seconds": 96.05033469200134, + "variant": "all_four_combined" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2.toml new file mode 100644 index 000000000..1b0e6fde6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2.toml @@ -0,0 +1,35 @@ +dataset = "homesite-insurance" +variant = "all_four_combined" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2/report.json new file mode 100644 index 000000000..5deeb3675 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2/report.json @@ -0,0 +1,184 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 19, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_four_combined" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.9290865174356744, + "train_loss": 0.3692213460803032, + "val_score": 0.9257781416460449 + }, + { + "epoch": 1, + "test_score": 0.9399931707621637, + "train_loss": 0.26565207486802883, + "val_score": 0.9366023707359086 + }, + { + "epoch": 2, + "test_score": 0.9425727882610563, + "train_loss": 0.250053295628591, + "val_score": 0.9397225055674711 + }, + { + "epoch": 3, + "test_score": 0.9405813057684445, + "train_loss": 0.25021838085217907, + "val_score": 0.9381491225026873 + }, + { + "epoch": 4, + "test_score": 0.9423215363871476, + "train_loss": 0.2392362048680132, + "val_score": 0.9406268267254861 + }, + { + "epoch": 5, + "test_score": 0.9441924005834136, + "train_loss": 0.2352831008759412, + "val_score": 0.9422248276953261 + }, + { + "epoch": 6, + "test_score": 0.9442216490705383, + "train_loss": 0.2291740291497924, + "val_score": 0.9434997626218505 + }, + { + "epoch": 7, + "test_score": 0.9457917890447852, + "train_loss": 0.2204523051326925, + "val_score": 0.9440485748545299 + }, + { + "epoch": 8, + "test_score": 0.9475897313623712, + "train_loss": 0.21289140324700961, + "val_score": 0.9451809721107797 + }, + { + "epoch": 9, + "test_score": 0.9476547884062236, + "train_loss": 0.2199919426983053, + "val_score": 0.945304536409943 + }, + { + "epoch": 10, + "test_score": 0.9467265445979869, + "train_loss": 0.21338573572310535, + "val_score": 0.946054151618899 + }, + { + "epoch": 11, + "test_score": 0.9477642074677223, + "train_loss": 0.20716341110793027, + "val_score": 0.9463127451272946 + }, + { + "epoch": 12, + "test_score": 0.9476992953977839, + "train_loss": 0.20451430732553655, + "val_score": 0.9465820210480627 + }, + { + "epoch": 13, + "test_score": 0.9472819638243694, + "train_loss": 0.20352806760506195, + "val_score": 0.9467325507453713 + }, + { + "epoch": 14, + "test_score": 0.9486999415948425, + "train_loss": 0.20124661556699058, + "val_score": 0.9476719524180912 + }, + { + "epoch": 15, + "test_score": 0.947577468985489, + "train_loss": 0.19827090759168972, + "val_score": 0.9471636630155559 + }, + { + "epoch": 16, + "test_score": 0.9488903165060824, + "train_loss": 0.1969674747098576, + "val_score": 0.9474078726337114 + }, + { + "epoch": 17, + "test_score": 0.9474682794660205, + "train_loss": 0.19572840292345395, + "val_score": 0.947753525604916 + }, + { + "epoch": 18, + "test_score": 0.9480860374753701, + "train_loss": 0.19547649269754236, + "val_score": 0.9481652161064331 + }, + { + "epoch": 19, + "test_score": 0.9502789974647928, + "train_loss": 0.19355940371751784, + "val_score": 0.9485711389844801 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9502789974647928, + "score": 0.9502789974647928 + }, + "val": { + "roc-auc": 0.9485711389844801, + "score": 0.9485711389844801 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2", + "seed": 2, + "task_type": "binclass", + "time_seconds": 96.36672067642212, + "variant": "all_four_combined" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..7c1e38fdb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml @@ -0,0 +1,34 @@ +dataset = "homesite-insurance" +variant = "all_minus_cf_fisd" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..42f738bda --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0/report.json @@ -0,0 +1,183 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 19, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_minus_cf_fisd" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.8867273748152669, + "train_loss": 0.4190130883997137, + "val_score": 0.8861749045533049 + }, + { + "epoch": 1, + "test_score": 0.9354356379127661, + "train_loss": 0.2752422061833468, + "val_score": 0.9309266599725895 + }, + { + "epoch": 2, + "test_score": 0.9386020494092362, + "train_loss": 0.25981232632290235, + "val_score": 0.9352112305054106 + }, + { + "epoch": 3, + "test_score": 0.9428584231148728, + "train_loss": 0.2525121568278833, + "val_score": 0.939197440821062 + }, + { + "epoch": 4, + "test_score": 0.9422175055227813, + "train_loss": 0.24629227424209768, + "val_score": 0.9399979447566122 + }, + { + "epoch": 5, + "test_score": 0.9429763352314928, + "train_loss": 0.24511424750089644, + "val_score": 0.9412687298077129 + }, + { + "epoch": 6, + "test_score": 0.9424501886545541, + "train_loss": 0.2420458352023905, + "val_score": 0.9414001102917317 + }, + { + "epoch": 7, + "test_score": 0.9441893923747006, + "train_loss": 0.22993632825938137, + "val_score": 0.9436276438465077 + }, + { + "epoch": 8, + "test_score": 0.945197794676152, + "train_loss": 0.2305623028766025, + "val_score": 0.9442499756942149 + }, + { + "epoch": 9, + "test_score": 0.9449498047153054, + "train_loss": 0.22382904548536647, + "val_score": 0.9449283748206871 + }, + { + "epoch": 10, + "test_score": 0.9442909466013718, + "train_loss": 0.22422936938025734, + "val_score": 0.9454736702097868 + }, + { + "epoch": 11, + "test_score": 0.9444200700339982, + "train_loss": 0.21886501434174452, + "val_score": 0.9449596131837814 + }, + { + "epoch": 12, + "test_score": 0.9465477313563789, + "train_loss": 0.21386859565973282, + "val_score": 0.9465164934600688 + }, + { + "epoch": 13, + "test_score": 0.9456236676293256, + "train_loss": 0.21183434413238006, + "val_score": 0.9469095183142883 + }, + { + "epoch": 14, + "test_score": 0.9448259728305354, + "train_loss": 0.2103140194307674, + "val_score": 0.9473692928596451 + }, + { + "epoch": 15, + "test_score": 0.9467284896646648, + "train_loss": 0.20671901418404146, + "val_score": 0.9475178003718955 + }, + { + "epoch": 16, + "test_score": 0.9442787687926061, + "train_loss": 0.20530494559894907, + "val_score": 0.9480879774293199 + }, + { + "epoch": 17, + "test_score": 0.9446361174091539, + "train_loss": 0.1996095741336996, + "val_score": 0.9480463995461023 + }, + { + "epoch": 18, + "test_score": 0.944189694403688, + "train_loss": 0.2015354111790657, + "val_score": 0.948330235199289 + }, + { + "epoch": 19, + "test_score": 0.9438768769410195, + "train_loss": 0.19719995788552544, + "val_score": 0.9486877399333562 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9438768769410195, + "score": 0.9438768769410195 + }, + "val": { + "roc-auc": 0.9486877399333562, + "score": 0.9486877399333562 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0", + "seed": 0, + "task_type": "binclass", + "time_seconds": 94.5395917892456, + "variant": "all_minus_cf_fisd" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1.toml new file mode 100644 index 000000000..17983d828 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1.toml @@ -0,0 +1,34 @@ +dataset = "homesite-insurance" +variant = "all_minus_cf_fisd" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1/report.json new file mode 100644 index 000000000..9ec7484a7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1/report.json @@ -0,0 +1,183 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 19, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_minus_cf_fisd" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.9011766348635782, + "train_loss": 0.4144984155893326, + "val_score": 0.8989568989190753 + }, + { + "epoch": 1, + "test_score": 0.9314295979129168, + "train_loss": 0.2808895646171136, + "val_score": 0.925805326846914 + }, + { + "epoch": 2, + "test_score": 0.9368095194511018, + "train_loss": 0.2609179992567409, + "val_score": 0.9320041505084555 + }, + { + "epoch": 3, + "test_score": 0.9408773425005816, + "train_loss": 0.25344904268329793, + "val_score": 0.9373312013993098 + }, + { + "epoch": 4, + "test_score": 0.9423485377786072, + "train_loss": 0.23822286413474517, + "val_score": 0.9391652441180923 + }, + { + "epoch": 5, + "test_score": 0.9437661410331363, + "train_loss": 0.2377325576814738, + "val_score": 0.9408794855603414 + }, + { + "epoch": 6, + "test_score": 0.9432573067577464, + "train_loss": 0.2404585752974857, + "val_score": 0.9416476697033968 + }, + { + "epoch": 7, + "test_score": 0.9425979653974329, + "train_loss": 0.23373751518401234, + "val_score": 0.9408246315926113 + }, + { + "epoch": 8, + "test_score": 0.9462087340210961, + "train_loss": 0.22808001962575045, + "val_score": 0.9446144877391139 + }, + { + "epoch": 9, + "test_score": 0.9455337717215624, + "train_loss": 0.22486585839228196, + "val_score": 0.9451965341436188 + }, + { + "epoch": 10, + "test_score": 0.9457187463545103, + "train_loss": 0.22854889671910894, + "val_score": 0.9444836875342859 + }, + { + "epoch": 11, + "test_score": 0.9457179489979838, + "train_loss": 0.21878958547657187, + "val_score": 0.9453681209418593 + }, + { + "epoch": 12, + "test_score": 0.9440899885944188, + "train_loss": 0.21896708648313176, + "val_score": 0.9451183810871748 + }, + { + "epoch": 13, + "test_score": 0.9454167173672661, + "train_loss": 0.21345828541300513, + "val_score": 0.9444320778547578 + }, + { + "epoch": 14, + "test_score": 0.9462222528385651, + "train_loss": 0.213582714443857, + "val_score": 0.9465140316695633 + }, + { + "epoch": 15, + "test_score": 0.9449545163675065, + "train_loss": 0.2090706546198238, + "val_score": 0.9466988769680983 + }, + { + "epoch": 16, + "test_score": 0.9479850993878329, + "train_loss": 0.20644646869464353, + "val_score": 0.9477843859073254 + }, + { + "epoch": 17, + "test_score": 0.943974396060421, + "train_loss": 0.20588010414080185, + "val_score": 0.9470860462777578 + }, + { + "epoch": 18, + "test_score": 0.9452268136212463, + "train_loss": 0.19995117322965103, + "val_score": 0.9480954155534905 + }, + { + "epoch": 19, + "test_score": 0.9468154377695126, + "train_loss": 0.20072065158323807, + "val_score": 0.9483137412029014 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9468154377695126, + "score": 0.9468154377695126 + }, + "val": { + "roc-auc": 0.9483137412029014, + "score": 0.9483137412029014 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1", + "seed": 1, + "task_type": "binclass", + "time_seconds": 96.52392911911011, + "variant": "all_minus_cf_fisd" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2.toml new file mode 100644 index 000000000..a1d5dd1d2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2.toml @@ -0,0 +1,34 @@ +dataset = "homesite-insurance" +variant = "all_minus_cf_fisd" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2/report.json new file mode 100644 index 000000000..e6cf07544 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2/report.json @@ -0,0 +1,183 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 19, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_minus_cf_fisd" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.9040055712508612, + "train_loss": 0.4124534674666145, + "val_score": 0.9022017146476378 + }, + { + "epoch": 1, + "test_score": 0.9323568148225967, + "train_loss": 0.2793294776569713, + "val_score": 0.9278128027068161 + }, + { + "epoch": 2, + "test_score": 0.938795239230637, + "train_loss": 0.25687101740728724, + "val_score": 0.9354104772784034 + }, + { + "epoch": 3, + "test_score": 0.9392835959406918, + "train_loss": 0.2551576859571717, + "val_score": 0.9361047197852009 + }, + { + "epoch": 4, + "test_score": 0.9422998627870229, + "train_loss": 0.2425997487523339, + "val_score": 0.9403174060352589 + }, + { + "epoch": 5, + "test_score": 0.9433914559527208, + "train_loss": 0.24269262145866047, + "val_score": 0.9411395913114763 + }, + { + "epoch": 6, + "test_score": 0.9437721816128812, + "train_loss": 0.23879283395680515, + "val_score": 0.9426016223164263 + }, + { + "epoch": 7, + "test_score": 0.9451093968321654, + "train_loss": 0.22980331656607714, + "val_score": 0.943478301083907 + }, + { + "epoch": 8, + "test_score": 0.9471574433135084, + "train_loss": 0.22093958367000927, + "val_score": 0.9447153420208632 + }, + { + "epoch": 9, + "test_score": 0.9459051344831182, + "train_loss": 0.235467699711973, + "val_score": 0.9443288233272656 + }, + { + "epoch": 10, + "test_score": 0.9456907663891317, + "train_loss": 0.2297248977151784, + "val_score": 0.945183521822375 + }, + { + "epoch": 11, + "test_score": 0.9477198575312351, + "train_loss": 0.21863165281035685, + "val_score": 0.9454252344815891 + }, + { + "epoch": 12, + "test_score": 0.9457698617403113, + "train_loss": 0.21798922636292198, + "val_score": 0.9455848640116595 + }, + { + "epoch": 13, + "test_score": 0.946250172398146, + "train_loss": 0.21708055368878626, + "val_score": 0.9462267583017775 + }, + { + "epoch": 14, + "test_score": 0.9465536873680073, + "train_loss": 0.21340600766918877, + "val_score": 0.9464460686673908 + }, + { + "epoch": 15, + "test_score": 0.9450038799851819, + "train_loss": 0.2088709598237818, + "val_score": 0.9465693604111766 + }, + { + "epoch": 16, + "test_score": 0.9451183610525068, + "train_loss": 0.20932438238100573, + "val_score": 0.9471085189082304 + }, + { + "epoch": 17, + "test_score": 0.9446490321686482, + "train_loss": 0.2056827260689302, + "val_score": 0.9475481507319862 + }, + { + "epoch": 18, + "test_score": 0.9433471664220312, + "train_loss": 0.20389442484487186, + "val_score": 0.9475102743266357 + }, + { + "epoch": 19, + "test_score": 0.947036245121507, + "train_loss": 0.2016631698066538, + "val_score": 0.9483005706236965 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.947036245121507, + "score": 0.947036245121507 + }, + "val": { + "roc-auc": 0.9483005706236965, + "score": 0.9483005706236965 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2", + "seed": 2, + "task_type": "binclass", + "time_seconds": 95.87906002998352, + "variant": "all_minus_cf_fisd" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0.toml new file mode 100644 index 000000000..c1324daa6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0.toml @@ -0,0 +1,33 @@ +dataset = "homesite-insurance" +variant = "all_minus_esam" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0/report.json new file mode 100644 index 000000000..4f0fb2a36 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0/report.json @@ -0,0 +1,182 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 19, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_minus_esam" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.9282623648976015, + "train_loss": 0.37420019615780226, + "val_score": 0.9251574715069288 + }, + { + "epoch": 1, + "test_score": 0.9371021734585819, + "train_loss": 0.27270237613808024, + "val_score": 0.9346151255186971 + }, + { + "epoch": 2, + "test_score": 0.9409601467677244, + "train_loss": 0.254967031695626, + "val_score": 0.938594398860388 + }, + { + "epoch": 3, + "test_score": 0.9424858401562084, + "train_loss": 0.24498489864847878, + "val_score": 0.9402089202029422 + }, + { + "epoch": 4, + "test_score": 0.9423627935468052, + "train_loss": 0.23897522552446884, + "val_score": 0.9400627513916721 + }, + { + "epoch": 5, + "test_score": 0.9436719925572326, + "train_loss": 0.23254627517678522, + "val_score": 0.942443874297671 + }, + { + "epoch": 6, + "test_score": 0.9456386603482525, + "train_loss": 0.22714561522006987, + "val_score": 0.943583815183399 + }, + { + "epoch": 7, + "test_score": 0.9447918314738174, + "train_loss": 0.22587331425059926, + "val_score": 0.9446441698989243 + }, + { + "epoch": 8, + "test_score": 0.9470115995561479, + "train_loss": 0.21659111380577087, + "val_score": 0.9450748073952258 + }, + { + "epoch": 9, + "test_score": 0.9469706444254775, + "train_loss": 0.2131239863959226, + "val_score": 0.945244732484875 + }, + { + "epoch": 10, + "test_score": 0.9454978181909208, + "train_loss": 0.21075186621059072, + "val_score": 0.9459386496836415 + }, + { + "epoch": 11, + "test_score": 0.9478734332306693, + "train_loss": 0.2058184477415952, + "val_score": 0.9464268579093381 + }, + { + "epoch": 12, + "test_score": 0.9481618225888494, + "train_loss": 0.2057848500934514, + "val_score": 0.9464468423729783 + }, + { + "epoch": 13, + "test_score": 0.9474614777732278, + "train_loss": 0.19984769130294974, + "val_score": 0.9469739644728812 + }, + { + "epoch": 14, + "test_score": 0.9473789151292749, + "train_loss": 0.20040455880490216, + "val_score": 0.9474729166557132 + }, + { + "epoch": 15, + "test_score": 0.9472418060502253, + "train_loss": 0.19612910435958342, + "val_score": 0.9469740523939708 + }, + { + "epoch": 16, + "test_score": 0.946604150371196, + "train_loss": 0.19511878978122366, + "val_score": 0.9478675768421971 + }, + { + "epoch": 17, + "test_score": 0.945813837160854, + "train_loss": 0.1943703759800304, + "val_score": 0.9468067737291153 + }, + { + "epoch": 18, + "test_score": 0.9494150737496799, + "train_loss": 0.1947024403647943, + "val_score": 0.9476518712412524 + }, + { + "epoch": 19, + "test_score": 0.9485440704751056, + "train_loss": 0.19200101332231, + "val_score": 0.9489494986009732 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9485440704751056, + "score": 0.9485440704751056 + }, + "val": { + "roc-auc": 0.9489494986009732, + "score": 0.9489494986009732 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0", + "seed": 0, + "task_type": "binclass", + "time_seconds": 57.65527606010437, + "variant": "all_minus_esam" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1.toml new file mode 100644 index 000000000..cd4d3cafc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1.toml @@ -0,0 +1,33 @@ +dataset = "homesite-insurance" +variant = "all_minus_esam" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1/report.json new file mode 100644 index 000000000..666a0d0d6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1/report.json @@ -0,0 +1,182 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 19, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_minus_esam" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.9311408582011115, + "train_loss": 0.36883535276759755, + "val_score": 0.9270232273626889 + }, + { + "epoch": 1, + "test_score": 0.9384909269042493, + "train_loss": 0.2649405887181109, + "val_score": 0.9355450668821883 + }, + { + "epoch": 2, + "test_score": 0.9405701306959163, + "train_loss": 0.2457744756882841, + "val_score": 0.9378696829039744 + }, + { + "epoch": 3, + "test_score": 0.941262356972361, + "train_loss": 0.2503055283969099, + "val_score": 0.9383217731461104 + }, + { + "epoch": 4, + "test_score": 0.9439972536141755, + "train_loss": 0.23776601932265543, + "val_score": 0.9409290378863756 + }, + { + "epoch": 5, + "test_score": 0.9465358555766004, + "train_loss": 0.23235300061377612, + "val_score": 0.9434845962339142 + }, + { + "epoch": 6, + "test_score": 0.945467603211037, + "train_loss": 0.22437328885902058, + "val_score": 0.943037869178237 + }, + { + "epoch": 7, + "test_score": 0.9459721486748079, + "train_loss": 0.22132552550597626, + "val_score": 0.9439452763664928 + }, + { + "epoch": 8, + "test_score": 0.9467611450387656, + "train_loss": 0.2172819274392995, + "val_score": 0.9445786950635843 + }, + { + "epoch": 9, + "test_score": 0.9446675042615081, + "train_loss": 0.21471701684323224, + "val_score": 0.9449388022619001 + }, + { + "epoch": 10, + "test_score": 0.9474626617268577, + "train_loss": 0.2101835078813813, + "val_score": 0.945552992616721 + }, + { + "epoch": 11, + "test_score": 0.9461134015915622, + "train_loss": 0.20978831648826599, + "val_score": 0.9451966924015798 + }, + { + "epoch": 12, + "test_score": 0.9470384922171722, + "train_loss": 0.2077528560703451, + "val_score": 0.9460920192321405 + }, + { + "epoch": 13, + "test_score": 0.9475360910142365, + "train_loss": 0.2031910687685013, + "val_score": 0.9464087285806861 + }, + { + "epoch": 14, + "test_score": 0.9476712912700865, + "train_loss": 0.20127531723542647, + "val_score": 0.9469623237206334 + }, + { + "epoch": 15, + "test_score": 0.9477483811487906, + "train_loss": 0.1988604185256091, + "val_score": 0.9468381615580617 + }, + { + "epoch": 16, + "test_score": 0.9478177149231024, + "train_loss": 0.1974608377976851, + "val_score": 0.9479933391685973 + }, + { + "epoch": 17, + "test_score": 0.9474401786890472, + "train_loss": 0.196479739790613, + "val_score": 0.9479001955663963 + }, + { + "epoch": 18, + "test_score": 0.9478719472480521, + "train_loss": 0.19606405632062393, + "val_score": 0.947817444236972 + }, + { + "epoch": 19, + "test_score": 0.9512055136672225, + "train_loss": 0.19362684867598795, + "val_score": 0.9482717237142363 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9512055136672225, + "score": 0.9512055136672225 + }, + "val": { + "roc-auc": 0.9482717237142363, + "score": 0.9482717237142363 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1", + "seed": 1, + "task_type": "binclass", + "time_seconds": 60.01497411727905, + "variant": "all_minus_esam" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2.toml new file mode 100644 index 000000000..fd0766099 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2.toml @@ -0,0 +1,33 @@ +dataset = "homesite-insurance" +variant = "all_minus_esam" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2/report.json new file mode 100644 index 000000000..8ec991525 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2/report.json @@ -0,0 +1,182 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 18, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_minus_esam" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.9305516117281577, + "train_loss": 0.36503461572256957, + "val_score": 0.9267991868424613 + }, + { + "epoch": 1, + "test_score": 0.9385174812928078, + "train_loss": 0.2688400410793044, + "val_score": 0.9351827001118724 + }, + { + "epoch": 2, + "test_score": 0.9415470978203752, + "train_loss": 0.2523974738337777, + "val_score": 0.9381120285950332 + }, + { + "epoch": 3, + "test_score": 0.9416739379138582, + "train_loss": 0.24787599999796261, + "val_score": 0.939197282563101 + }, + { + "epoch": 4, + "test_score": 0.9444743627647454, + "train_loss": 0.23736426600001076, + "val_score": 0.9420454422964482 + }, + { + "epoch": 5, + "test_score": 0.9440574902753915, + "train_loss": 0.23959832096641714, + "val_score": 0.9421820365010734 + }, + { + "epoch": 6, + "test_score": 0.9437026182965391, + "train_loss": 0.23264405889944598, + "val_score": 0.942717256133318 + }, + { + "epoch": 7, + "test_score": 0.9459018363265774, + "train_loss": 0.2235704476183111, + "val_score": 0.943923375223102 + }, + { + "epoch": 8, + "test_score": 0.9460808911913753, + "train_loss": 0.22024327313358133, + "val_score": 0.9448243114191718 + }, + { + "epoch": 9, + "test_score": 0.9465984118204385, + "train_loss": 0.2157839154655283, + "val_score": 0.9454886519634352 + }, + { + "epoch": 10, + "test_score": 0.9481819014759215, + "train_loss": 0.21046820960261606, + "val_score": 0.9453150341880276 + }, + { + "epoch": 11, + "test_score": 0.9455237564403451, + "train_loss": 0.21228812594305385, + "val_score": 0.9457623591071136 + }, + { + "epoch": 12, + "test_score": 0.947202747661595, + "train_loss": 0.2047921211882071, + "val_score": 0.9457797674828315 + }, + { + "epoch": 13, + "test_score": 0.9466076055828101, + "train_loss": 0.2042220937934789, + "val_score": 0.9464812986479477 + }, + { + "epoch": 14, + "test_score": 0.9488007467896252, + "train_loss": 0.199715626781637, + "val_score": 0.9472975668348425 + }, + { + "epoch": 15, + "test_score": 0.9474458326716885, + "train_loss": 0.19854097799821332, + "val_score": 0.9470691302601407 + }, + { + "epoch": 16, + "test_score": 0.9493707842189902, + "train_loss": 0.19561608921397816, + "val_score": 0.9477912789207413 + }, + { + "epoch": 17, + "test_score": 0.9485832617564904, + "train_loss": 0.19515144513411956, + "val_score": 0.9479748933240233 + }, + { + "epoch": 18, + "test_score": 0.9485803381158939, + "train_loss": 0.19575764618136665, + "val_score": 0.9482433076181145 + }, + { + "epoch": 19, + "test_score": 0.9506953021396505, + "train_loss": 0.19345392286777496, + "val_score": 0.9479477696679169 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9485803381158939, + "score": 0.9485803381158939 + }, + "val": { + "roc-auc": 0.9482433076181145, + "score": 0.9482433076181145 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2", + "seed": 2, + "task_type": "binclass", + "time_seconds": 60.87337589263916, + "variant": "all_minus_esam" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0.toml new file mode 100644 index 000000000..84687163b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0.toml @@ -0,0 +1,33 @@ +dataset = "homesite-insurance" +variant = "all_minus_mfb" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0/report.json new file mode 100644 index 000000000..eb71a0c05 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0/report.json @@ -0,0 +1,182 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 16, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_minus_mfb" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.936080856397636, + "train_loss": 0.3485633905638348, + "val_score": 0.9330192082786779 + }, + { + "epoch": 1, + "test_score": 0.9443432701031219, + "train_loss": 0.21431942704049023, + "val_score": 0.9408591757886702 + }, + { + "epoch": 2, + "test_score": 0.9482861860446371, + "train_loss": 0.20039536370472474, + "val_score": 0.9447542646871785 + }, + { + "epoch": 3, + "test_score": 0.9473031058534765, + "train_loss": 0.1941048334945332, + "val_score": 0.9441183578232547 + }, + { + "epoch": 4, + "test_score": 0.9496835171135424, + "train_loss": 0.1905539946122603, + "val_score": 0.9447491564718793 + }, + { + "epoch": 5, + "test_score": 0.9511852052381201, + "train_loss": 0.18824819949540225, + "val_score": 0.9475763470253842 + }, + { + "epoch": 6, + "test_score": 0.9501237545653494, + "train_loss": 0.18603328303857283, + "val_score": 0.947620676838703 + }, + { + "epoch": 7, + "test_score": 0.9511574668959317, + "train_loss": 0.18426710014993494, + "val_score": 0.9483009486843812 + }, + { + "epoch": 8, + "test_score": 0.9534666597208054, + "train_loss": 0.18335880569436333, + "val_score": 0.9493766016694772 + }, + { + "epoch": 9, + "test_score": 0.9512971975865901, + "train_loss": 0.18194586864926598, + "val_score": 0.9486100880271222 + }, + { + "epoch": 10, + "test_score": 0.9522303946702174, + "train_loss": 0.18081731620160016, + "val_score": 0.9494002876109848 + }, + { + "epoch": 11, + "test_score": 0.9537785469341931, + "train_loss": 0.18012406162240288, + "val_score": 0.9505937636481007 + }, + { + "epoch": 12, + "test_score": 0.9533768967057965, + "train_loss": 0.179088133438067, + "val_score": 0.9504195392171756 + }, + { + "epoch": 13, + "test_score": 0.9528421120998224, + "train_loss": 0.17811415845697576, + "val_score": 0.950407643493768 + }, + { + "epoch": 14, + "test_score": 0.9529662460135799, + "train_loss": 0.1776683739640496, + "val_score": 0.9504370443060922 + }, + { + "epoch": 15, + "test_score": 0.9527797249922172, + "train_loss": 0.17663268406282773, + "val_score": 0.9503563942907068 + }, + { + "epoch": 16, + "test_score": 0.9532358612499129, + "train_loss": 0.17589410096406938, + "val_score": 0.951222742330173 + }, + { + "epoch": 17, + "test_score": 0.9526458536639111, + "train_loss": 0.17520266324281691, + "val_score": 0.9507549757577818 + }, + { + "epoch": 18, + "test_score": 0.9533179285663269, + "train_loss": 0.17484911680221557, + "val_score": 0.9511791246776792 + }, + { + "epoch": 19, + "test_score": 0.9527198990904239, + "train_loss": 0.17424605746160854, + "val_score": 0.9510088391115631 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9532358612499129, + "score": 0.9532358612499129 + }, + "val": { + "roc-auc": 0.951222742330173, + "score": 0.951222742330173 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0", + "seed": 0, + "task_type": "binclass", + "time_seconds": 93.82961058616638, + "variant": "all_minus_mfb" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1.toml new file mode 100644 index 000000000..327c2f2a0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1.toml @@ -0,0 +1,33 @@ +dataset = "homesite-insurance" +variant = "all_minus_mfb" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1/report.json new file mode 100644 index 000000000..e4b0b5126 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1/report.json @@ -0,0 +1,182 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 16, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_minus_mfb" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.9349781123217293, + "train_loss": 0.3451001604849642, + "val_score": 0.9310866675633446 + }, + { + "epoch": 1, + "test_score": 0.9438002461463759, + "train_loss": 0.21620039181275802, + "val_score": 0.9403393247628676 + }, + { + "epoch": 2, + "test_score": 0.9468972997062153, + "train_loss": 0.20234638845378702, + "val_score": 0.9439125081764416 + }, + { + "epoch": 3, + "test_score": 0.9498045461693108, + "train_loss": 0.19468857971104708, + "val_score": 0.9458391669708881 + }, + { + "epoch": 4, + "test_score": 0.951420727442373, + "train_loss": 0.1909984537146308, + "val_score": 0.9467180877261508 + }, + { + "epoch": 5, + "test_score": 0.9516745526032531, + "train_loss": 0.18896782167933204, + "val_score": 0.9476140300043379 + }, + { + "epoch": 6, + "test_score": 0.9515406450314684, + "train_loss": 0.18661685626615176, + "val_score": 0.9481888580873961 + }, + { + "epoch": 7, + "test_score": 0.9531164269071972, + "train_loss": 0.18516999808224766, + "val_score": 0.9489969935735135 + }, + { + "epoch": 8, + "test_score": 0.9522627600964907, + "train_loss": 0.18365479057485407, + "val_score": 0.9487103796138988 + }, + { + "epoch": 9, + "test_score": 0.9513448577607774, + "train_loss": 0.1828564101999456, + "val_score": 0.9483731846515029 + }, + { + "epoch": 10, + "test_score": 0.9536176258897894, + "train_loss": 0.18148549510674042, + "val_score": 0.9493563534425687 + }, + { + "epoch": 11, + "test_score": 0.9524729722716125, + "train_loss": 0.1805111982605674, + "val_score": 0.949499594481559 + }, + { + "epoch": 12, + "test_score": 0.9531607647625244, + "train_loss": 0.17996388402852145, + "val_score": 0.9504649240835682 + }, + { + "epoch": 13, + "test_score": 0.9535825301214718, + "train_loss": 0.1787733332677321, + "val_score": 0.9495215132091678 + }, + { + "epoch": 14, + "test_score": 0.9532390990006563, + "train_loss": 0.1780598908662796, + "val_score": 0.9504280060180932 + }, + { + "epoch": 15, + "test_score": 0.9536520330320163, + "train_loss": 0.17685881514440882, + "val_score": 0.9504281554839451 + }, + { + "epoch": 16, + "test_score": 0.9528538308245276, + "train_loss": 0.17636417136950927, + "val_score": 0.9509531674777006 + }, + { + "epoch": 17, + "test_score": 0.9524469373729123, + "train_loss": 0.17591779868711124, + "val_score": 0.9503000896250001 + }, + { + "epoch": 18, + "test_score": 0.952751672539882, + "train_loss": 0.17490007105198774, + "val_score": 0.9499999797781494 + }, + { + "epoch": 19, + "test_score": 0.9532202282295332, + "train_loss": 0.17446433955972845, + "val_score": 0.9509216917276646 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9528538308245276, + "score": 0.9528538308245276 + }, + "val": { + "roc-auc": 0.9509531674777006, + "score": 0.9509531674777006 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1", + "seed": 1, + "task_type": "binclass", + "time_seconds": 94.35714793205261, + "variant": "all_minus_mfb" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2.toml new file mode 100644 index 000000000..cb4193e6e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2.toml @@ -0,0 +1,33 @@ +dataset = "homesite-insurance" +variant = "all_minus_mfb" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2/report.json new file mode 100644 index 000000000..c8b69b870 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2/report.json @@ -0,0 +1,182 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 19, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_minus_mfb" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.9348914662458687, + "train_loss": 0.34214188944209706, + "val_score": 0.9314159760040135 + }, + { + "epoch": 1, + "test_score": 0.9444708350661741, + "train_loss": 0.2145986951210282, + "val_score": 0.9404227355004626 + }, + { + "epoch": 2, + "test_score": 0.9474591823529247, + "train_loss": 0.20296300189061597, + "val_score": 0.9442555410991793 + }, + { + "epoch": 3, + "test_score": 0.9483235893144174, + "train_loss": 0.1946356772021814, + "val_score": 0.9448229574343938 + }, + { + "epoch": 4, + "test_score": 0.9502820177546653, + "train_loss": 0.1902028499679132, + "val_score": 0.9466603587387945 + }, + { + "epoch": 5, + "test_score": 0.9518550934506681, + "train_loss": 0.1883919584480199, + "val_score": 0.946771587709103 + }, + { + "epoch": 6, + "test_score": 0.948750694545859, + "train_loss": 0.1861147192391482, + "val_score": 0.9466200732955922 + }, + { + "epoch": 7, + "test_score": 0.9516568537046007, + "train_loss": 0.18476087220690468, + "val_score": 0.9477665027577242 + }, + { + "epoch": 8, + "test_score": 0.9528440571665004, + "train_loss": 0.18319841162724929, + "val_score": 0.9497558229126473 + }, + { + "epoch": 9, + "test_score": 0.9525012663471377, + "train_loss": 0.18201071254231713, + "val_score": 0.949482827929794 + }, + { + "epoch": 10, + "test_score": 0.9528152315199576, + "train_loss": 0.18100855905901303, + "val_score": 0.9496849497224129 + }, + { + "epoch": 11, + "test_score": 0.9535917480461624, + "train_loss": 0.17989628274332392, + "val_score": 0.9499481151274618 + }, + { + "epoch": 12, + "test_score": 0.9539670855091903, + "train_loss": 0.17919402501799844, + "val_score": 0.9502128543200125 + }, + { + "epoch": 13, + "test_score": 0.9540179955152801, + "train_loss": 0.17842414839701218, + "val_score": 0.9505214397598896 + }, + { + "epoch": 14, + "test_score": 0.9538338182388588, + "train_loss": 0.17782833643934942, + "val_score": 0.9511898422584875 + }, + { + "epoch": 15, + "test_score": 0.952540469709682, + "train_loss": 0.17649481784213672, + "val_score": 0.9504172268925222 + }, + { + "epoch": 16, + "test_score": 0.9527322701977416, + "train_loss": 0.1761845365166664, + "val_score": 0.9497746204415795 + }, + { + "epoch": 17, + "test_score": 0.952729129096274, + "train_loss": 0.17521973590959203, + "val_score": 0.9507788111651413 + }, + { + "epoch": 18, + "test_score": 0.952827276435969, + "train_loss": 0.1749107297171246, + "val_score": 0.9509241359339523 + }, + { + "epoch": 19, + "test_score": 0.953869240198483, + "train_loss": 0.17429973889480938, + "val_score": 0.9512909251350691 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.953869240198483, + "score": 0.953869240198483 + }, + "val": { + "roc-auc": 0.9512909251350691, + "score": 0.9512909251350691 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2", + "seed": 2, + "task_type": "binclass", + "time_seconds": 96.05296754837036, + "variant": "all_minus_mfb" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0.toml new file mode 100644 index 000000000..bba72ccae --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0.toml @@ -0,0 +1,32 @@ +dataset = "homesite-insurance" +variant = "all_minus_rla" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0/report.json new file mode 100644 index 000000000..9c2d3f6fe --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0/report.json @@ -0,0 +1,181 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 17, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_minus_rla" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.9290964481487749, + "train_loss": 0.3734414360739968, + "val_score": 0.9260200916922008 + }, + { + "epoch": 1, + "test_score": 0.9399295755386096, + "train_loss": 0.2582675423134457, + "val_score": 0.936778230499098 + }, + { + "epoch": 2, + "test_score": 0.939959705950377, + "train_loss": 0.2561210263859142, + "val_score": 0.9372083404688625 + }, + { + "epoch": 3, + "test_score": 0.9446191917047086, + "train_loss": 0.2515790993517095, + "val_score": 0.9414589031242706 + }, + { + "epoch": 4, + "test_score": 0.9436923009863349, + "train_loss": 0.2490062885663726, + "val_score": 0.9410485314390957 + }, + { + "epoch": 5, + "test_score": 0.9448191590565831, + "train_loss": 0.24894793981855565, + "val_score": 0.9423545728470804 + }, + { + "epoch": 6, + "test_score": 0.9446520041338828, + "train_loss": 0.24585373239083724, + "val_score": 0.9425746129577363 + }, + { + "epoch": 7, + "test_score": 0.9464069737671635, + "train_loss": 0.233190402794968, + "val_score": 0.9439372755473496 + }, + { + "epoch": 8, + "test_score": 0.9476272312814273, + "train_loss": 0.23624674583023245, + "val_score": 0.9448485776398697 + }, + { + "epoch": 9, + "test_score": 0.9479388889527848, + "train_loss": 0.23320516537536273, + "val_score": 0.945495896661209 + }, + { + "epoch": 10, + "test_score": 0.948785524528668, + "train_loss": 0.23475631827657872, + "val_score": 0.9462668503185829 + }, + { + "epoch": 11, + "test_score": 0.9475035081270926, + "train_loss": 0.23386801467700438, + "val_score": 0.9448978925789625 + }, + { + "epoch": 12, + "test_score": 0.9484173149497389, + "train_loss": 0.22608610892837697, + "val_score": 0.945860417498217 + }, + { + "epoch": 13, + "test_score": 0.9491997150006151, + "train_loss": 0.225085971030322, + "val_score": 0.9468609331202384 + }, + { + "epoch": 14, + "test_score": 0.948916822570003, + "train_loss": 0.23035850606181404, + "val_score": 0.946784046127483 + }, + { + "epoch": 15, + "test_score": 0.9502937606416895, + "train_loss": 0.2231764554977417, + "val_score": 0.9476667299053756 + }, + { + "epoch": 16, + "test_score": 0.949525338492343, + "train_loss": 0.22973006882450797, + "val_score": 0.9472551185328391 + }, + { + "epoch": 17, + "test_score": 0.950880361340715, + "train_loss": 0.21746040094982494, + "val_score": 0.9482617446705797 + }, + { + "epoch": 18, + "test_score": 0.9492371424327144, + "train_loss": 0.22736059007319537, + "val_score": 0.9470035675037112 + }, + { + "epoch": 19, + "test_score": 0.9503750910073745, + "train_loss": 0.21671231402592225, + "val_score": 0.9481422335336418 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.950880361340715, + "score": 0.950880361340715 + }, + "val": { + "roc-auc": 0.9482617446705797, + "score": 0.9482617446705797 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0", + "seed": 0, + "task_type": "binclass", + "time_seconds": 78.73098039627075, + "variant": "all_minus_rla" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1.toml new file mode 100644 index 000000000..a6f61b0b5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1.toml @@ -0,0 +1,32 @@ +dataset = "homesite-insurance" +variant = "all_minus_rla" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1/report.json new file mode 100644 index 000000000..95758417a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1/report.json @@ -0,0 +1,181 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 19, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_minus_rla" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.9305360632758944, + "train_loss": 0.37063259875232524, + "val_score": 0.9251245714352433 + }, + { + "epoch": 1, + "test_score": 0.9385317853856436, + "train_loss": 0.26606528745456176, + "val_score": 0.935285717252423 + }, + { + "epoch": 2, + "test_score": 0.9417092028184089, + "train_loss": 0.2536956027150154, + "val_score": 0.9385187603471032 + }, + { + "epoch": 3, + "test_score": 0.9429347639616886, + "train_loss": 0.2526001357219436, + "val_score": 0.940356513335862 + }, + { + "epoch": 4, + "test_score": 0.9453012698071818, + "train_loss": 0.23765619004314595, + "val_score": 0.9422723050836483 + }, + { + "epoch": 5, + "test_score": 0.9459483004659751, + "train_loss": 0.23879074142737822, + "val_score": 0.9423621780213208 + }, + { + "epoch": 6, + "test_score": 0.9443578520626261, + "train_loss": 0.24308968741785397, + "val_score": 0.9423130125480803 + }, + { + "epoch": 7, + "test_score": 0.9453902354656645, + "train_loss": 0.23823609148914163, + "val_score": 0.942570225695371 + }, + { + "epoch": 8, + "test_score": 0.9479785030747518, + "train_loss": 0.23380982577800752, + "val_score": 0.945337735413333 + }, + { + "epoch": 9, + "test_score": 0.9481873138353728, + "train_loss": 0.23272585475986654, + "val_score": 0.9454698104739585 + }, + { + "epoch": 10, + "test_score": 0.9478268845231552, + "train_loss": 0.23764072954654694, + "val_score": 0.9451820535401805 + }, + { + "epoch": 11, + "test_score": 0.9490152840198445, + "train_loss": 0.22946590293537486, + "val_score": 0.9461689062248957 + }, + { + "epoch": 12, + "test_score": 0.9480973333594931, + "train_loss": 0.2318623957308856, + "val_score": 0.9453694661345283 + }, + { + "epoch": 13, + "test_score": 0.9469292785353847, + "train_loss": 0.22568691576069053, + "val_score": 0.9441586168901303 + }, + { + "epoch": 14, + "test_score": 0.9497359493457278, + "train_loss": 0.2304897647012364, + "val_score": 0.9472400488581012 + }, + { + "epoch": 15, + "test_score": 0.9498432058796782, + "train_loss": 0.2239592286673459, + "val_score": 0.946994010481284 + }, + { + "epoch": 16, + "test_score": 0.9489725529587292, + "train_loss": 0.22322151389988987, + "val_score": 0.9466023659881697 + }, + { + "epoch": 17, + "test_score": 0.9489942144776944, + "train_loss": 0.22959728931838816, + "val_score": 0.9468746751865251 + }, + { + "epoch": 18, + "test_score": 0.9492767323923624, + "train_loss": 0.21688801876523278, + "val_score": 0.9471568491311207 + }, + { + "epoch": 19, + "test_score": 0.9499504624136281, + "train_loss": 0.22332347401163793, + "val_score": 0.9477251798456656 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9499504624136281, + "score": 0.9499504624136281 + }, + "val": { + "roc-auc": 0.9477251798456656, + "score": 0.9477251798456656 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1", + "seed": 1, + "task_type": "binclass", + "time_seconds": 80.56382298469543, + "variant": "all_minus_rla" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2.toml new file mode 100644 index 000000000..4a1e240b2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2.toml @@ -0,0 +1,32 @@ +dataset = "homesite-insurance" +variant = "all_minus_rla" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2/report.json new file mode 100644 index 000000000..e6ea21568 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2/report.json @@ -0,0 +1,181 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 18, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_minus_rla" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.9311648634650177, + "train_loss": 0.3705499173565344, + "val_score": 0.9267300896582344 + }, + { + "epoch": 1, + "test_score": 0.9400797443510673, + "train_loss": 0.2669092017141255, + "val_score": 0.9365076005935516 + }, + { + "epoch": 2, + "test_score": 0.9425562129102363, + "train_loss": 0.25113668482411994, + "val_score": 0.9396609959732668 + }, + { + "epoch": 3, + "test_score": 0.941631702180282, + "train_loss": 0.25384827730330556, + "val_score": 0.9383607573571884 + }, + { + "epoch": 4, + "test_score": 0.9429357425356073, + "train_loss": 0.24375565146858041, + "val_score": 0.9402041460877832 + }, + { + "epoch": 5, + "test_score": 0.9448555475089664, + "train_loss": 0.2422500709241087, + "val_score": 0.9419154509656434 + }, + { + "epoch": 6, + "test_score": 0.9459835049647285, + "train_loss": 0.2398611607876691, + "val_score": 0.9434383937013894 + }, + { + "epoch": 7, + "test_score": 0.9476710134034183, + "train_loss": 0.2345308242873712, + "val_score": 0.9442095232009424 + }, + { + "epoch": 8, + "test_score": 0.9479903184487326, + "train_loss": 0.22430998127568852, + "val_score": 0.9445603019716639 + }, + { + "epoch": 9, + "test_score": 0.9462158135805571, + "train_loss": 0.24261990758505733, + "val_score": 0.9435801928345121 + }, + { + "epoch": 10, + "test_score": 0.9463019522477192, + "train_loss": 0.23785935233939778, + "val_score": 0.9441950689738309 + }, + { + "epoch": 11, + "test_score": 0.9477470280589279, + "train_loss": 0.22844018380750308, + "val_score": 0.9446325467308944 + }, + { + "epoch": 12, + "test_score": 0.9483520042015373, + "train_loss": 0.22788471349261025, + "val_score": 0.9456674307067956 + }, + { + "epoch": 13, + "test_score": 0.9496501247887126, + "train_loss": 0.2299680548635396, + "val_score": 0.9467883454687589 + }, + { + "epoch": 14, + "test_score": 0.9485763271709432, + "train_loss": 0.2274209422144023, + "val_score": 0.9462431819612933 + }, + { + "epoch": 15, + "test_score": 0.948765687264786, + "train_loss": 0.2249194474382834, + "val_score": 0.9465975391203569 + }, + { + "epoch": 16, + "test_score": 0.9479952113183261, + "train_loss": 0.22566421208056536, + "val_score": 0.9461114058323722 + }, + { + "epoch": 17, + "test_score": 0.9493630643580763, + "train_loss": 0.2237590559504249, + "val_score": 0.9475332481073184 + }, + { + "epoch": 18, + "test_score": 0.9505799995534802, + "train_loss": 0.2211470286954533, + "val_score": 0.9482112603809969 + }, + { + "epoch": 19, + "test_score": 0.9499611663209362, + "train_loss": 0.22110925899310546, + "val_score": 0.947609044878564 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9505799995534802, + "score": 0.9505799995534802 + }, + "val": { + "roc-auc": 0.9482112603809969, + "score": 0.9482112603809969 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2", + "seed": 2, + "task_type": "binclass", + "time_seconds": 80.3422474861145, + "variant": "all_minus_rla" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..49c31352e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0.toml @@ -0,0 +1,27 @@ +dataset = "homesite-insurance" +variant = "baseline_plr" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..cd049b235 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0/report.json @@ -0,0 +1,176 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 18, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "baseline_plr" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.9092551612404702, + "train_loss": 0.40265455543994905, + "val_score": 0.9069948735146987 + }, + { + "epoch": 1, + "test_score": 0.9408753853527443, + "train_loss": 0.2279508112506433, + "val_score": 0.9375135849073393 + }, + { + "epoch": 2, + "test_score": 0.9445531560869376, + "train_loss": 0.20344534069299697, + "val_score": 0.9413967956666578 + }, + { + "epoch": 3, + "test_score": 0.9475919542957172, + "train_loss": 0.19599832499569111, + "val_score": 0.9441133463211538 + }, + { + "epoch": 4, + "test_score": 0.948740111450146, + "train_loss": 0.1918256477876143, + "val_score": 0.9462975611551404 + }, + { + "epoch": 5, + "test_score": 0.9494191813439062, + "train_loss": 0.18962883813814682, + "val_score": 0.9473306163723804 + }, + { + "epoch": 6, + "test_score": 0.9488655259668092, + "train_loss": 0.18693935193798758, + "val_score": 0.948109166411886 + }, + { + "epoch": 7, + "test_score": 0.9504254211178088, + "train_loss": 0.18496789133006877, + "val_score": 0.9483060041470267 + }, + { + "epoch": 8, + "test_score": 0.9500466526054857, + "train_loss": 0.18423595116897062, + "val_score": 0.94922840188104 + }, + { + "epoch": 9, + "test_score": 0.9496331628407889, + "train_loss": 0.18239941827275535, + "val_score": 0.9492990552685507 + }, + { + "epoch": 10, + "test_score": 0.9507528205399816, + "train_loss": 0.18131620423360303, + "val_score": 0.9500436413911881 + }, + { + "epoch": 11, + "test_score": 0.9509186465351379, + "train_loss": 0.18082551035014066, + "val_score": 0.9500871007857209 + }, + { + "epoch": 12, + "test_score": 0.95122214942384, + "train_loss": 0.17972185178236527, + "val_score": 0.9503499496748476 + }, + { + "epoch": 13, + "test_score": 0.950892768691511, + "train_loss": 0.17814393598924982, + "val_score": 0.9501892738838125 + }, + { + "epoch": 14, + "test_score": 0.950397549882866, + "train_loss": 0.17763228077780116, + "val_score": 0.9497813024443802 + }, + { + "epoch": 15, + "test_score": 0.9527338407484751, + "train_loss": 0.17669161016290838, + "val_score": 0.9512763390263235 + }, + { + "epoch": 16, + "test_score": 0.952465481952729, + "train_loss": 0.1758088152516972, + "val_score": 0.9514608590168274 + }, + { + "epoch": 17, + "test_score": 0.9532766593255099, + "train_loss": 0.17514954602176494, + "val_score": 0.951688504301724 + }, + { + "epoch": 18, + "test_score": 0.9527329104991943, + "train_loss": 0.17444596791809255, + "val_score": 0.951942736924081 + }, + { + "epoch": 19, + "test_score": 0.9520153379501403, + "train_loss": 0.1736760964447802, + "val_score": 0.9514972319715476 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9527329104991943, + "score": 0.9527329104991943 + }, + "val": { + "roc-auc": 0.951942736924081, + "score": 0.951942736924081 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0", + "seed": 0, + "task_type": "binclass", + "time_seconds": 49.509262800216675, + "variant": "baseline_plr" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1.toml new file mode 100644 index 000000000..2fd6f359c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1.toml @@ -0,0 +1,27 @@ +dataset = "homesite-insurance" +variant = "baseline_plr" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1/report.json new file mode 100644 index 000000000..d63534675 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1/report.json @@ -0,0 +1,176 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 18, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "baseline_plr" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.9152291617154047, + "train_loss": 0.40116534287279304, + "val_score": 0.9128087083095999 + }, + { + "epoch": 1, + "test_score": 0.9410798227336301, + "train_loss": 0.22753327705643395, + "val_score": 0.93512569207745 + }, + { + "epoch": 2, + "test_score": 0.9450291054461962, + "train_loss": 0.20542177815328944, + "val_score": 0.940434516926454 + }, + { + "epoch": 3, + "test_score": 0.9471194118234347, + "train_loss": 0.1974578472700986, + "val_score": 0.9431667175348782 + }, + { + "epoch": 4, + "test_score": 0.9472607855517837, + "train_loss": 0.19329350929368627, + "val_score": 0.9452731485809968 + }, + { + "epoch": 5, + "test_score": 0.9492152876151975, + "train_loss": 0.18974924954501066, + "val_score": 0.9464897126962115 + }, + { + "epoch": 6, + "test_score": 0.9491863894816981, + "train_loss": 0.18798113004727798, + "val_score": 0.9469293533120763 + }, + { + "epoch": 7, + "test_score": 0.9498860094277503, + "train_loss": 0.18644553978334774, + "val_score": 0.9478204423461236 + }, + { + "epoch": 8, + "test_score": 0.9500854331274479, + "train_loss": 0.18484502827579324, + "val_score": 0.948568923373025 + }, + { + "epoch": 9, + "test_score": 0.9513454618187518, + "train_loss": 0.18316087357022545, + "val_score": 0.9494386124138916 + }, + { + "epoch": 10, + "test_score": 0.9506907596236825, + "train_loss": 0.18158665922555056, + "val_score": 0.9491904111782734 + }, + { + "epoch": 11, + "test_score": 0.951191439116417, + "train_loss": 0.18065576593984256, + "val_score": 0.9498100526406422 + }, + { + "epoch": 12, + "test_score": 0.9526199758202842, + "train_loss": 0.1797984992915934, + "val_score": 0.9495069886451847 + }, + { + "epoch": 13, + "test_score": 0.9513438308622209, + "train_loss": 0.17902052050287073, + "val_score": 0.9501305777644716 + }, + { + "epoch": 14, + "test_score": 0.9505508718779504, + "train_loss": 0.17793472490527412, + "val_score": 0.9501276587843008 + }, + { + "epoch": 15, + "test_score": 0.9529368767148602, + "train_loss": 0.17711421264843508, + "val_score": 0.950880439152478 + }, + { + "epoch": 16, + "test_score": 0.9520506995039669, + "train_loss": 0.17653414783152668, + "val_score": 0.9504092700339237 + }, + { + "epoch": 17, + "test_score": 0.9520371806864979, + "train_loss": 0.17552702819759194, + "val_score": 0.9506431840925006 + }, + { + "epoch": 18, + "test_score": 0.9530616509300706, + "train_loss": 0.1746248097582297, + "val_score": 0.9512467711639294 + }, + { + "epoch": 19, + "test_score": 0.9537073043366819, + "train_loss": 0.17431941289793362, + "val_score": 0.9502714801024816 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9530616509300706, + "score": 0.9530616509300706 + }, + "val": { + "roc-auc": 0.9512467711639294, + "score": 0.9512467711639294 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1", + "seed": 1, + "task_type": "binclass", + "time_seconds": 49.59510111808777, + "variant": "baseline_plr" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2.toml new file mode 100644 index 000000000..cfcdc9bd4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2.toml @@ -0,0 +1,27 @@ +dataset = "homesite-insurance" +variant = "baseline_plr" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2/report.json new file mode 100644 index 000000000..f86492c8b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2/report.json @@ -0,0 +1,176 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 19, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "baseline_plr" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.918776262628557, + "train_loss": 0.4011460320516066, + "val_score": 0.914550108576391 + }, + { + "epoch": 1, + "test_score": 0.9399168057530289, + "train_loss": 0.22690394493666563, + "val_score": 0.9361857126928352 + }, + { + "epoch": 2, + "test_score": 0.9448195456536869, + "train_loss": 0.2040758810260079, + "val_score": 0.9417620638329265 + }, + { + "epoch": 3, + "test_score": 0.9486874255136111, + "train_loss": 0.195921021158045, + "val_score": 0.9447484706873813 + }, + { + "epoch": 4, + "test_score": 0.9473506452160686, + "train_loss": 0.19198633405295285, + "val_score": 0.9453271760904858 + }, + { + "epoch": 5, + "test_score": 0.9478740252074842, + "train_loss": 0.18984181637113745, + "val_score": 0.9463248342770988 + }, + { + "epoch": 6, + "test_score": 0.950552043750421, + "train_loss": 0.18705180775035513, + "val_score": 0.9478500541690625 + }, + { + "epoch": 7, + "test_score": 0.949526160011188, + "train_loss": 0.1852110438726165, + "val_score": 0.9479885210928937 + }, + { + "epoch": 8, + "test_score": 0.9500179477705382, + "train_loss": 0.1838859125971794, + "val_score": 0.9489083691153118 + }, + { + "epoch": 9, + "test_score": 0.951933753880106, + "train_loss": 0.182511460239237, + "val_score": 0.949106499290468 + }, + { + "epoch": 10, + "test_score": 0.9505244141386678, + "train_loss": 0.18200897046110848, + "val_score": 0.9494714861092504 + }, + { + "epoch": 11, + "test_score": 0.951459894561439, + "train_loss": 0.18012812530452554, + "val_score": 0.9500796011167878 + }, + { + "epoch": 12, + "test_score": 0.9524452822540621, + "train_loss": 0.1792988423596729, + "val_score": 0.9499561423229318 + }, + { + "epoch": 13, + "test_score": 0.9514597375063656, + "train_loss": 0.1785095995122736, + "val_score": 0.9502315639278549 + }, + { + "epoch": 14, + "test_score": 0.9517478489978773, + "train_loss": 0.17744325155561622, + "val_score": 0.950572829636694 + }, + { + "epoch": 15, + "test_score": 0.9520442844082779, + "train_loss": 0.1771064200184562, + "val_score": 0.9507174422446804 + }, + { + "epoch": 16, + "test_score": 0.9534308632452372, + "train_loss": 0.17622807310386138, + "val_score": 0.9508445145953139 + }, + { + "epoch": 17, + "test_score": 0.9525583498257268, + "train_loss": 0.17589684643528677, + "val_score": 0.950965230251178 + }, + { + "epoch": 18, + "test_score": 0.9526653768176468, + "train_loss": 0.17428164116360925, + "val_score": 0.9513501136125109 + }, + { + "epoch": 19, + "test_score": 0.9538975705174861, + "train_loss": 0.17425130158662797, + "val_score": 0.9515441114964615 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9538975705174861, + "score": 0.9538975705174861 + }, + "val": { + "roc-auc": 0.9515441114964615, + "score": 0.9515441114964615 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2", + "seed": 2, + "task_type": "binclass", + "time_seconds": 52.342146158218384, + "variant": "baseline_plr" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..6f754cff6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0.toml @@ -0,0 +1,28 @@ +dataset = "homesite-insurance" +variant = "cf_fisd_only" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..d0b26f2fa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,177 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 17, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "cf_fisd_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.9356782275953207, + "train_loss": 0.34595361920920287, + "val_score": 0.932453427275728 + }, + { + "epoch": 1, + "test_score": 0.9450704230116513, + "train_loss": 0.2139470558274876, + "val_score": 0.9414894732870849 + }, + { + "epoch": 2, + "test_score": 0.9472626218880262, + "train_loss": 0.20079242763194172, + "val_score": 0.943445313091132 + }, + { + "epoch": 3, + "test_score": 0.9502677378241484, + "train_loss": 0.19390834854407743, + "val_score": 0.9455957662267558 + }, + { + "epoch": 4, + "test_score": 0.9512641556153858, + "train_loss": 0.1904226701367985, + "val_score": 0.9471411200482117 + }, + { + "epoch": 5, + "test_score": 0.9516697322206167, + "train_loss": 0.1884030981497331, + "val_score": 0.9477306837058675 + }, + { + "epoch": 6, + "test_score": 0.951924825903243, + "train_loss": 0.18619168563322588, + "val_score": 0.9484983930750396 + }, + { + "epoch": 7, + "test_score": 0.9525656226837398, + "train_loss": 0.18428844687613574, + "val_score": 0.9486409219532052 + }, + { + "epoch": 8, + "test_score": 0.952576954811341, + "train_loss": 0.18353629532185467, + "val_score": 0.9492243926793594 + }, + { + "epoch": 9, + "test_score": 0.9527248523658148, + "train_loss": 0.18191406997767362, + "val_score": 0.9493485196734955 + }, + { + "epoch": 10, + "test_score": 0.9529742920658001, + "train_loss": 0.180889469249682, + "val_score": 0.9500480550298802 + }, + { + "epoch": 11, + "test_score": 0.9533969514305494, + "train_loss": 0.18059069622646678, + "val_score": 0.9499987664671146 + }, + { + "epoch": 12, + "test_score": 0.9535607719532305, + "train_loss": 0.17952959659424694, + "val_score": 0.950490429991628 + }, + { + "epoch": 13, + "test_score": 0.9526063482723798, + "train_loss": 0.17814569906754926, + "val_score": 0.9500473164927284 + }, + { + "epoch": 14, + "test_score": 0.9527289841223603, + "train_loss": 0.1777255663817579, + "val_score": 0.9496370063523164 + }, + { + "epoch": 15, + "test_score": 0.9539749141005397, + "train_loss": 0.1767724638635462, + "val_score": 0.9511459608427252 + }, + { + "epoch": 16, + "test_score": 0.9541068403621681, + "train_loss": 0.17599785151806743, + "val_score": 0.9512804889017473 + }, + { + "epoch": 17, + "test_score": 0.9543305592735997, + "train_loss": 0.17548072730953043, + "val_score": 0.9515030083871268 + }, + { + "epoch": 18, + "test_score": 0.9539477556540068, + "train_loss": 0.17482344752008264, + "val_score": 0.9514199141654539 + }, + { + "epoch": 19, + "test_score": 0.9534808913266842, + "train_loss": 0.17422030784867026, + "val_score": 0.9511812171996087 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9543305592735997, + "score": 0.9543305592735997 + }, + "val": { + "roc-auc": 0.9515030083871268, + "score": 0.9515030083871268 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0", + "seed": 0, + "task_type": "binclass", + "time_seconds": 48.676718950271606, + "variant": "cf_fisd_only" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1.toml new file mode 100644 index 000000000..c0b676360 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1.toml @@ -0,0 +1,28 @@ +dataset = "homesite-insurance" +variant = "cf_fisd_only" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1/report.json new file mode 100644 index 000000000..62a65e68f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1/report.json @@ -0,0 +1,177 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 18, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "cf_fisd_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.9341219205486897, + "train_loss": 0.3445870617573911, + "val_score": 0.9301370054994816 + }, + { + "epoch": 1, + "test_score": 0.9455012854836944, + "train_loss": 0.21479442458261144, + "val_score": 0.9419169807926006 + }, + { + "epoch": 2, + "test_score": 0.9477954010215248, + "train_loss": 0.20204473598436876, + "val_score": 0.9440465087089268 + }, + { + "epoch": 3, + "test_score": 0.9502180359340077, + "train_loss": 0.1942431166768074, + "val_score": 0.9464708799988436 + }, + { + "epoch": 4, + "test_score": 0.950256188235676, + "train_loss": 0.19089977063915947, + "val_score": 0.9466997913474287 + }, + { + "epoch": 5, + "test_score": 0.9516093505854868, + "train_loss": 0.18810367069461129, + "val_score": 0.9472306852620709 + }, + { + "epoch": 6, + "test_score": 0.9514332314424452, + "train_loss": 0.18654819618571888, + "val_score": 0.9474986775349324 + }, + { + "epoch": 7, + "test_score": 0.9522986048966968, + "train_loss": 0.1853742083365267, + "val_score": 0.9484116852965887 + }, + { + "epoch": 8, + "test_score": 0.9523285178475936, + "train_loss": 0.18406797484918075, + "val_score": 0.9490725529578176 + }, + { + "epoch": 9, + "test_score": 0.9534142516509388, + "train_loss": 0.1827931050549854, + "val_score": 0.9496698185029124 + }, + { + "epoch": 10, + "test_score": 0.9527738656298648, + "train_loss": 0.1812525972723961, + "val_score": 0.9496054163048641 + }, + { + "epoch": 11, + "test_score": 0.9529002466392873, + "train_loss": 0.1802022703669288, + "val_score": 0.9498561408757505 + }, + { + "epoch": 12, + "test_score": 0.9535008010775234, + "train_loss": 0.17959909669377588, + "val_score": 0.9494651645829163 + }, + { + "epoch": 13, + "test_score": 0.9532634787805064, + "train_loss": 0.17891152690757406, + "val_score": 0.9501905047790651 + }, + { + "epoch": 14, + "test_score": 0.9519700698455323, + "train_loss": 0.17768442129546946, + "val_score": 0.9499346456165526 + }, + { + "epoch": 15, + "test_score": 0.9539713018338523, + "train_loss": 0.17704763520847666, + "val_score": 0.950807570153512 + }, + { + "epoch": 16, + "test_score": 0.9529998074746422, + "train_loss": 0.1763513352383267, + "val_score": 0.950307307946447 + }, + { + "epoch": 17, + "test_score": 0.9530448339560608, + "train_loss": 0.17554833618077365, + "val_score": 0.950576091509114 + }, + { + "epoch": 18, + "test_score": 0.9539302138104276, + "train_loss": 0.1747353514487093, + "val_score": 0.9512915581669135 + }, + { + "epoch": 19, + "test_score": 0.95444112604525, + "train_loss": 0.17438724718310616, + "val_score": 0.9504254123459533 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9539302138104276, + "score": 0.9539302138104276 + }, + "val": { + "roc-auc": 0.9512915581669135, + "score": 0.9512915581669135 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1", + "seed": 1, + "task_type": "binclass", + "time_seconds": 51.1213116645813, + "variant": "cf_fisd_only" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2.toml new file mode 100644 index 000000000..f2eb7bc69 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2.toml @@ -0,0 +1,28 @@ +dataset = "homesite-insurance" +variant = "cf_fisd_only" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2/report.json new file mode 100644 index 000000000..40dcc2465 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2/report.json @@ -0,0 +1,177 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 19, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "cf_fisd_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.9368268196714911, + "train_loss": 0.3421837103637782, + "val_score": 0.9326621695263851 + }, + { + "epoch": 1, + "test_score": 0.9439968428547529, + "train_loss": 0.214590031450445, + "val_score": 0.9407243400058346 + }, + { + "epoch": 2, + "test_score": 0.9473855356046752, + "train_loss": 0.20220571661537343, + "val_score": 0.9441511875580686 + }, + { + "epoch": 3, + "test_score": 0.9500893957477605, + "train_loss": 0.19425439482385462, + "val_score": 0.9462104753160047 + }, + { + "epoch": 4, + "test_score": 0.9496412572176471, + "train_loss": 0.1903872162103653, + "val_score": 0.946274886306162 + }, + { + "epoch": 5, + "test_score": 0.9502430439341512, + "train_loss": 0.188402750546282, + "val_score": 0.9469396224953281 + }, + { + "epoch": 6, + "test_score": 0.952653790985696, + "train_loss": 0.18608629364858975, + "val_score": 0.9482615512441828 + }, + { + "epoch": 7, + "test_score": 0.9516714115017857, + "train_loss": 0.18428249873898245, + "val_score": 0.9480207002116454 + }, + { + "epoch": 8, + "test_score": 0.9526286863362763, + "train_loss": 0.18304257894104178, + "val_score": 0.949256809185053 + }, + { + "epoch": 9, + "test_score": 0.9536667358031155, + "train_loss": 0.18201136128468948, + "val_score": 0.9492468037650695 + }, + { + "epoch": 10, + "test_score": 0.9526991678207396, + "train_loss": 0.18154333640228618, + "val_score": 0.9496464139088913 + }, + { + "epoch": 11, + "test_score": 0.9533547640216111, + "train_loss": 0.1797881559892134, + "val_score": 0.9503523675048084 + }, + { + "epoch": 12, + "test_score": 0.9540047062398416, + "train_loss": 0.17908011282032185, + "val_score": 0.9503211643101501 + }, + { + "epoch": 13, + "test_score": 0.9527204306614416, + "train_loss": 0.17852983230894262, + "val_score": 0.9501864076562951 + }, + { + "epoch": 14, + "test_score": 0.9532465409949018, + "train_loss": 0.17740027660673316, + "val_score": 0.9507598993387929 + }, + { + "epoch": 15, + "test_score": 0.9535459966951747, + "train_loss": 0.17713124508207495, + "val_score": 0.9510151518457881 + }, + { + "epoch": 16, + "test_score": 0.9538396534388924, + "train_loss": 0.17623664411631498, + "val_score": 0.9507649284251114 + }, + { + "epoch": 17, + "test_score": 0.9536572279305969, + "train_loss": 0.17600064846602353, + "val_score": 0.9508994740683517 + }, + { + "epoch": 18, + "test_score": 0.9538222082445892, + "train_loss": 0.17448357275941156, + "val_score": 0.9514048005301712 + }, + { + "epoch": 19, + "test_score": 0.9548333529692444, + "train_loss": 0.17447115494446322, + "val_score": 0.9519185850007992 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9548333529692444, + "score": 0.9548333529692444 + }, + "val": { + "roc-auc": 0.9519185850007992, + "score": 0.9519185850007992 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2", + "seed": 2, + "task_type": "binclass", + "time_seconds": 51.285852909088135, + "variant": "cf_fisd_only" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0.toml new file mode 100644 index 000000000..a9656bec3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0.toml @@ -0,0 +1,29 @@ +dataset = "homesite-insurance" +variant = "esam_only" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0/report.json new file mode 100644 index 000000000..b764b9767 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 18, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "esam_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.908729026744691, + "train_loss": 0.4032717401331121, + "val_score": 0.906570592713167 + }, + { + "epoch": 1, + "test_score": 0.9408769679846375, + "train_loss": 0.22863993658260864, + "val_score": 0.9373667566878979 + }, + { + "epoch": 2, + "test_score": 0.9445106787301715, + "train_loss": 0.20407469245520504, + "val_score": 0.941151777174479 + }, + { + "epoch": 3, + "test_score": 0.9475020342256348, + "train_loss": 0.19658909020098772, + "val_score": 0.9440126678815838 + }, + { + "epoch": 4, + "test_score": 0.9488865955089594, + "train_loss": 0.19217103773897345, + "val_score": 0.9462030196076161 + }, + { + "epoch": 5, + "test_score": 0.9495634062258951, + "train_loss": 0.19011524509299885, + "val_score": 0.9472272739237988 + }, + { + "epoch": 6, + "test_score": 0.9486842602498249, + "train_loss": 0.18731397566470234, + "val_score": 0.9479469871702204 + }, + { + "epoch": 7, + "test_score": 0.9506030141623083, + "train_loss": 0.18545044728300789, + "val_score": 0.9481834773167194 + }, + { + "epoch": 8, + "test_score": 0.949899769868409, + "train_loss": 0.18469505269419062, + "val_score": 0.9491123987955724 + }, + { + "epoch": 9, + "test_score": 0.9496426103075098, + "train_loss": 0.1828580309044231, + "val_score": 0.9492144839725745 + }, + { + "epoch": 10, + "test_score": 0.9505282559473858, + "train_loss": 0.18168031546202573, + "val_score": 0.9500062925123746 + }, + { + "epoch": 11, + "test_score": 0.9508295963085389, + "train_loss": 0.18124051933938806, + "val_score": 0.9500708969289285 + }, + { + "epoch": 12, + "test_score": 0.9511067743507127, + "train_loss": 0.179936251857064, + "val_score": 0.9503749104721527 + }, + { + "epoch": 13, + "test_score": 0.9508392612361309, + "train_loss": 0.17850419987331737, + "val_score": 0.9502711284181236 + }, + { + "epoch": 14, + "test_score": 0.9506935503715248, + "train_loss": 0.1779331474141641, + "val_score": 0.9501258651940752 + }, + { + "epoch": 15, + "test_score": 0.9527863213052987, + "train_loss": 0.17700313343243165, + "val_score": 0.9514039740719301 + }, + { + "epoch": 16, + "test_score": 0.9525207290950757, + "train_loss": 0.17610342638059095, + "val_score": 0.9515501956358539 + }, + { + "epoch": 17, + "test_score": 0.9533450266070624, + "train_loss": 0.1753809163516218, + "val_score": 0.9517171665768963 + }, + { + "epoch": 18, + "test_score": 0.9528578176071593, + "train_loss": 0.1747145946730267, + "val_score": 0.9520053367397949 + }, + { + "epoch": 19, + "test_score": 0.9522033207918011, + "train_loss": 0.17411214180968024, + "val_score": 0.9517029585288354 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9528578176071593, + "score": 0.9528578176071593 + }, + "val": { + "roc-auc": 0.9520053367397949, + "score": 0.9520053367397949 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0", + "seed": 0, + "task_type": "binclass", + "time_seconds": 77.46188020706177, + "variant": "esam_only" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1.toml new file mode 100644 index 000000000..9cf6fb745 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1.toml @@ -0,0 +1,29 @@ +dataset = "homesite-insurance" +variant = "esam_only" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1/report.json new file mode 100644 index 000000000..46458166c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1/report.json @@ -0,0 +1,178 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 18, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "esam_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.9147756953939565, + "train_loss": 0.40172804539853874, + "val_score": 0.9124704495020212 + }, + { + "epoch": 1, + "test_score": 0.9407769722275406, + "train_loss": 0.2284326516769149, + "val_score": 0.9349377167881294 + }, + { + "epoch": 2, + "test_score": 0.9448971187787707, + "train_loss": 0.2062601380727508, + "val_score": 0.9402407915978808 + }, + { + "epoch": 3, + "test_score": 0.9468855568191913, + "train_loss": 0.1979634551839395, + "val_score": 0.9429383864654839 + }, + { + "epoch": 4, + "test_score": 0.9470005090517362, + "train_loss": 0.19388176703994925, + "val_score": 0.9450562296690174 + }, + { + "epoch": 5, + "test_score": 0.9489158319149248, + "train_loss": 0.1902060573751276, + "val_score": 0.9463056059348284 + }, + { + "epoch": 6, + "test_score": 0.9488005051664354, + "train_loss": 0.18836823295463215, + "val_score": 0.9467129091739801 + }, + { + "epoch": 7, + "test_score": 0.9496745287308819, + "train_loss": 0.1869170370427045, + "val_score": 0.9476899938256536 + }, + { + "epoch": 8, + "test_score": 0.9498384579839986, + "train_loss": 0.1851530686020851, + "val_score": 0.948475076402108 + }, + { + "epoch": 9, + "test_score": 0.9512796194995327, + "train_loss": 0.18356213136152788, + "val_score": 0.9493400177041422 + }, + { + "epoch": 10, + "test_score": 0.9506040048173865, + "train_loss": 0.18194633613933217, + "val_score": 0.9490656775286198 + }, + { + "epoch": 11, + "test_score": 0.9510479511851571, + "train_loss": 0.18100393509322946, + "val_score": 0.949754521680523 + }, + { + "epoch": 12, + "test_score": 0.9524267135119263, + "train_loss": 0.1802269911224192, + "val_score": 0.94947667345353 + }, + { + "epoch": 13, + "test_score": 0.9511907988149638, + "train_loss": 0.1793379318985072, + "val_score": 0.9500892372681953 + }, + { + "epoch": 14, + "test_score": 0.9504332013845201, + "train_loss": 0.1783846991983327, + "val_score": 0.9500476242165418 + }, + { + "epoch": 15, + "test_score": 0.9528141804590822, + "train_loss": 0.17759880057790062, + "val_score": 0.9508749528764942 + }, + { + "epoch": 16, + "test_score": 0.9519126601756369, + "train_loss": 0.17679053274067966, + "val_score": 0.9503236700612003 + }, + { + "epoch": 17, + "test_score": 0.9519582786338702, + "train_loss": 0.1758375184102492, + "val_score": 0.9505821668563976 + }, + { + "epoch": 18, + "test_score": 0.9530825030113498, + "train_loss": 0.17506359707225452, + "val_score": 0.9512828188106186 + }, + { + "epoch": 19, + "test_score": 0.9535977644635883, + "train_loss": 0.17463966445489362, + "val_score": 0.9501736415141018 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9530825030113498, + "score": 0.9530825030113498 + }, + "val": { + "roc-auc": 0.9512828188106186, + "score": 0.9512828188106186 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1", + "seed": 1, + "task_type": "binclass", + "time_seconds": 79.07510232925415, + "variant": "esam_only" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2.toml new file mode 100644 index 000000000..91757e813 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2.toml @@ -0,0 +1,29 @@ +dataset = "homesite-insurance" +variant = "esam_only" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2/report.json new file mode 100644 index 000000000..0da573087 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2/report.json @@ -0,0 +1,178 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 19, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "esam_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.918368837605924, + "train_loss": 0.4018080147829923, + "val_score": 0.914369043884704 + }, + { + "epoch": 1, + "test_score": 0.9396808727893531, + "train_loss": 0.2275132194161415, + "val_score": 0.9359316471205482 + }, + { + "epoch": 2, + "test_score": 0.9447300000995488, + "train_loss": 0.20466342500664972, + "val_score": 0.941619904223337 + }, + { + "epoch": 3, + "test_score": 0.9486153493160954, + "train_loss": 0.19632872586900538, + "val_score": 0.9446383407306913 + }, + { + "epoch": 4, + "test_score": 0.9472420114299367, + "train_loss": 0.19241265126250007, + "val_score": 0.9452525047091854 + }, + { + "epoch": 5, + "test_score": 0.9477903873403365, + "train_loss": 0.19022366553544998, + "val_score": 0.9463075050303612 + }, + { + "epoch": 6, + "test_score": 0.9503154221606545, + "train_loss": 0.18738075386394154, + "val_score": 0.9476912071366883 + }, + { + "epoch": 7, + "test_score": 0.9493462353029068, + "train_loss": 0.18546505800702356, + "val_score": 0.947911519802722 + }, + { + "epoch": 8, + "test_score": 0.9497630473864636, + "train_loss": 0.18423645306717265, + "val_score": 0.9488656218816038 + }, + { + "epoch": 9, + "test_score": 0.9517127894730382, + "train_loss": 0.18286606940356168, + "val_score": 0.9490535971709243 + }, + { + "epoch": 10, + "test_score": 0.9501892102874651, + "train_loss": 0.18241622298955917, + "val_score": 0.9494314995977523 + }, + { + "epoch": 11, + "test_score": 0.951262440090738, + "train_loss": 0.18050547662106428, + "val_score": 0.9499718274452961 + }, + { + "epoch": 12, + "test_score": 0.9523709831231999, + "train_loss": 0.17971274432810871, + "val_score": 0.9499982301484686 + }, + { + "epoch": 13, + "test_score": 0.9512103582121777, + "train_loss": 0.1789512035521594, + "val_score": 0.950264877228661 + }, + { + "epoch": 14, + "test_score": 0.9516913091714654, + "train_loss": 0.17783631357279692, + "val_score": 0.9505823339064675 + }, + { + "epoch": 15, + "test_score": 0.9519843739383681, + "train_loss": 0.17752045812931927, + "val_score": 0.9507157629518711 + }, + { + "epoch": 16, + "test_score": 0.9534914985847162, + "train_loss": 0.17668290869756179, + "val_score": 0.9509019094825304 + }, + { + "epoch": 17, + "test_score": 0.9527285492006186, + "train_loss": 0.1762797315012325, + "val_score": 0.9509802647574802 + }, + { + "epoch": 18, + "test_score": 0.9528286295258319, + "train_loss": 0.17472596371715718, + "val_score": 0.9513271750002642 + }, + { + "epoch": 19, + "test_score": 0.9538855376826346, + "train_loss": 0.1745066287842664, + "val_score": 0.9515687469857353 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9538855376826346, + "score": 0.9538855376826346 + }, + "val": { + "roc-auc": 0.9515687469857353, + "score": 0.9515687469857353 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2", + "seed": 2, + "task_type": "binclass", + "time_seconds": 81.17848920822144, + "variant": "esam_only" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0.toml new file mode 100644 index 000000000..8d18b8743 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0.toml @@ -0,0 +1,29 @@ +dataset = "homesite-insurance" +variant = "mfb_only" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..ec843e55b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 19, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "mfb_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.8930970574258102, + "train_loss": 0.41482925062829795, + "val_score": 0.8924687005757092 + }, + { + "epoch": 1, + "test_score": 0.9331467414358352, + "train_loss": 0.2881732252511111, + "val_score": 0.929492192229193 + }, + { + "epoch": 2, + "test_score": 0.9378768898920224, + "train_loss": 0.26185889582742344, + "val_score": 0.9349895374782722 + }, + { + "epoch": 3, + "test_score": 0.9417591946563775, + "train_loss": 0.25147163529287686, + "val_score": 0.9388787268716761 + }, + { + "epoch": 4, + "test_score": 0.9420589523856374, + "train_loss": 0.24695172946561467, + "val_score": 0.9379653234651171 + }, + { + "epoch": 5, + "test_score": 0.9444818651647885, + "train_loss": 0.2433497359806841, + "val_score": 0.9416113231250033 + }, + { + "epoch": 6, + "test_score": 0.945757369821399, + "train_loss": 0.23861975249919024, + "val_score": 0.9428739227226728 + }, + { + "epoch": 7, + "test_score": 0.9469385085412347, + "train_loss": 0.24340633235194467, + "val_score": 0.9440592660590111 + }, + { + "epoch": 8, + "test_score": 0.9472159886123956, + "train_loss": 0.23544722199440002, + "val_score": 0.9436778204122771 + }, + { + "epoch": 9, + "test_score": 0.9472175591631293, + "train_loss": 0.23263336379419675, + "val_score": 0.9444757746362322 + }, + { + "epoch": 10, + "test_score": 0.9475347500055331, + "train_loss": 0.2365114538507028, + "val_score": 0.9451331694144264 + }, + { + "epoch": 11, + "test_score": 0.9489926560081199, + "train_loss": 0.23266613673080097, + "val_score": 0.9459791637216766 + }, + { + "epoch": 12, + "test_score": 0.9478042444302713, + "train_loss": 0.23365027592940765, + "val_score": 0.9451919358706387 + }, + { + "epoch": 13, + "test_score": 0.9493018491229417, + "train_loss": 0.22537394247271797, + "val_score": 0.9461747793536734 + }, + { + "epoch": 14, + "test_score": 0.94857972197676, + "train_loss": 0.23251519352197647, + "val_score": 0.9461889874017343 + }, + { + "epoch": 15, + "test_score": 0.9505958621158904, + "train_loss": 0.22507083226333965, + "val_score": 0.9471473624455652 + }, + { + "epoch": 16, + "test_score": 0.949072403741912, + "train_loss": 0.22468203807418996, + "val_score": 0.9466362507760575 + }, + { + "epoch": 17, + "test_score": 0.9498181737172153, + "train_loss": 0.2172338836572387, + "val_score": 0.9474565105804152 + }, + { + "epoch": 18, + "test_score": 0.9495097054719629, + "train_loss": 0.2258679667657072, + "val_score": 0.9473183865488329 + }, + { + "epoch": 19, + "test_score": 0.9503995432741815, + "train_loss": 0.21923035166480326, + "val_score": 0.947966382562561 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9503995432741815, + "score": 0.9503995432741815 + }, + "val": { + "roc-auc": 0.947966382562561, + "score": 0.947966382562561 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0", + "seed": 0, + "task_type": "binclass", + "time_seconds": 49.44327640533447, + "variant": "mfb_only" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1.toml new file mode 100644 index 000000000..bb63e7362 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1.toml @@ -0,0 +1,29 @@ +dataset = "homesite-insurance" +variant = "mfb_only" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1/report.json new file mode 100644 index 000000000..dba6eeee0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1/report.json @@ -0,0 +1,178 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 18, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "mfb_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.90697903454912, + "train_loss": 0.4116514666513963, + "val_score": 0.9035914305564858 + }, + { + "epoch": 1, + "test_score": 0.9317266253001322, + "train_loss": 0.2784177227453752, + "val_score": 0.9259784610563293 + }, + { + "epoch": 2, + "test_score": 0.9383684722716611, + "train_loss": 0.2549135125496171, + "val_score": 0.9332698273442572 + }, + { + "epoch": 3, + "test_score": 0.9389794889940154, + "train_loss": 0.2599698143926534, + "val_score": 0.9349251968249865 + }, + { + "epoch": 4, + "test_score": 0.9427043762502187, + "train_loss": 0.24525119445540688, + "val_score": 0.9390834335443257 + }, + { + "epoch": 5, + "test_score": 0.9455191535185797, + "train_loss": 0.24000001563267273, + "val_score": 0.9416237551670565 + }, + { + "epoch": 6, + "test_score": 0.9438849229932397, + "train_loss": 0.23375196429816159, + "val_score": 0.9414440620443656 + }, + { + "epoch": 7, + "test_score": 0.9461117223103932, + "train_loss": 0.2322191279042851, + "val_score": 0.9432047873666256 + }, + { + "epoch": 8, + "test_score": 0.9456184606495858, + "train_loss": 0.23114837624809959, + "val_score": 0.9428914541879162 + }, + { + "epoch": 9, + "test_score": 0.9455694473855355, + "train_loss": 0.23292614072561263, + "val_score": 0.9433439488670636 + }, + { + "epoch": 10, + "test_score": 0.9469385568658729, + "train_loss": 0.2312833772464232, + "val_score": 0.9443709814896741 + }, + { + "epoch": 11, + "test_score": 0.9480216448952896, + "train_loss": 0.23427447879856284, + "val_score": 0.9445003046201987 + }, + { + "epoch": 12, + "test_score": 0.947081042060895, + "train_loss": 0.2386595448309725, + "val_score": 0.9442177350307004 + }, + { + "epoch": 13, + "test_score": 0.9476110062842326, + "train_loss": 0.22934468537569047, + "val_score": 0.945519327631557 + }, + { + "epoch": 14, + "test_score": 0.9486105893392562, + "train_loss": 0.2297188869931481, + "val_score": 0.9461029829919994 + }, + { + "epoch": 15, + "test_score": 0.9489645189876684, + "train_loss": 0.22718335647474636, + "val_score": 0.9460358376559592 + }, + { + "epoch": 16, + "test_score": 0.948593832771044, + "train_loss": 0.2275947011329911, + "val_score": 0.9461941044091424 + }, + { + "epoch": 17, + "test_score": 0.9492380243573573, + "train_loss": 0.21871563467112454, + "val_score": 0.9470392370897156 + }, + { + "epoch": 18, + "test_score": 0.94973945288198, + "train_loss": 0.2260531865737655, + "val_score": 0.947404892108778 + }, + { + "epoch": 19, + "test_score": 0.9498296266564115, + "train_loss": 0.2279391365972432, + "val_score": 0.9473486665720519 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.94973945288198, + "score": 0.94973945288198 + }, + "val": { + "roc-auc": 0.947404892108778, + "score": 0.947404892108778 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1", + "seed": 1, + "task_type": "binclass", + "time_seconds": 48.79517936706543, + "variant": "mfb_only" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2.toml new file mode 100644 index 000000000..bf20c6844 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2.toml @@ -0,0 +1,29 @@ +dataset = "homesite-insurance" +variant = "mfb_only" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2/report.json new file mode 100644 index 000000000..6b2883abf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2/report.json @@ -0,0 +1,178 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 17, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "mfb_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.9094044602094448, + "train_loss": 0.41223958161744206, + "val_score": 0.9054979114608956 + }, + { + "epoch": 1, + "test_score": 0.933507545263997, + "train_loss": 0.28018381636251105, + "val_score": 0.9294203958675189 + }, + { + "epoch": 2, + "test_score": 0.9390422868610431, + "train_loss": 0.25977433242581105, + "val_score": 0.9353415559363555 + }, + { + "epoch": 3, + "test_score": 0.9399967951100104, + "train_loss": 0.2551980449394746, + "val_score": 0.9367911197308166 + }, + { + "epoch": 4, + "test_score": 0.9444567242718902, + "train_loss": 0.24537605155598033, + "val_score": 0.941659556634695 + }, + { + "epoch": 5, + "test_score": 0.9435413589796697, + "train_loss": 0.24892445301467722, + "val_score": 0.940676598854244 + }, + { + "epoch": 6, + "test_score": 0.9447349050503016, + "train_loss": 0.2462358137423342, + "val_score": 0.9421798824343809 + }, + { + "epoch": 7, + "test_score": 0.946477829767571, + "train_loss": 0.2382623243060979, + "val_score": 0.9432349706766461 + }, + { + "epoch": 8, + "test_score": 0.9459221809991583, + "train_loss": 0.23777699958194387, + "val_score": 0.9431933048723389 + }, + { + "epoch": 9, + "test_score": 0.9456133261568024, + "train_loss": 0.23771833492950958, + "val_score": 0.9430304574303932 + }, + { + "epoch": 10, + "test_score": 0.9482286555631468, + "train_loss": 0.23080746843056246, + "val_score": 0.9454318109790827 + }, + { + "epoch": 11, + "test_score": 0.9440564029710373, + "train_loss": 0.24243825890801168, + "val_score": 0.9417709878235093 + }, + { + "epoch": 12, + "test_score": 0.9483229127694859, + "train_loss": 0.23124837455424396, + "val_score": 0.9450030022414426 + }, + { + "epoch": 13, + "test_score": 0.9462020652210578, + "train_loss": 0.23264152123169465, + "val_score": 0.9434262957594763 + }, + { + "epoch": 14, + "test_score": 0.9489400063150637, + "train_loss": 0.2291149674491449, + "val_score": 0.9462765040542086 + }, + { + "epoch": 15, + "test_score": 0.9497078244064356, + "train_loss": 0.22526589808138933, + "val_score": 0.9460425812035226 + }, + { + "epoch": 16, + "test_score": 0.9500673476316916, + "train_loss": 0.21986289972608739, + "val_score": 0.9468494154575158 + }, + { + "epoch": 17, + "test_score": 0.9499664578687927, + "train_loss": 0.22433200790123506, + "val_score": 0.947465100470858 + }, + { + "epoch": 18, + "test_score": 0.9495361752924052, + "train_loss": 0.23154993870041587, + "val_score": 0.9463413282734864 + }, + { + "epoch": 19, + "test_score": 0.9493388053898209, + "train_loss": 0.2206502684138038, + "val_score": 0.946505186807961 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9499664578687927, + "score": 0.9499664578687927 + }, + "val": { + "roc-auc": 0.947465100470858, + "score": 0.947465100470858 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2", + "seed": 2, + "task_type": "binclass", + "time_seconds": 52.592193603515625, + "variant": "mfb_only" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0.toml new file mode 100644 index 000000000..a58c4224a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0.toml @@ -0,0 +1,30 @@ +dataset = "homesite-insurance" +variant = "rla_only" +seed = 0 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0/report.json new file mode 100644 index 000000000..260031bc0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 19, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "rla_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.9070250516856164, + "train_loss": 0.4060501190749082, + "val_score": 0.9046584057300496 + }, + { + "epoch": 1, + "test_score": 0.940496737652016, + "train_loss": 0.22928613221103494, + "val_score": 0.9369488765416809 + }, + { + "epoch": 2, + "test_score": 0.9447281154386683, + "train_loss": 0.2031014254147356, + "val_score": 0.9417829099232438 + }, + { + "epoch": 3, + "test_score": 0.944822408888486, + "train_loss": 0.1961885085160082, + "val_score": 0.9434031109681789 + }, + { + "epoch": 4, + "test_score": 0.9469906629067522, + "train_loss": 0.19204557131637226, + "val_score": 0.9441429757283105 + }, + { + "epoch": 5, + "test_score": 0.9480114121532017, + "train_loss": 0.18933298194950277, + "val_score": 0.9470159907536556 + }, + { + "epoch": 6, + "test_score": 0.9461172675625991, + "train_loss": 0.18728381584991108, + "val_score": 0.9473758781492477 + }, + { + "epoch": 7, + "test_score": 0.9469529334456656, + "train_loss": 0.18507160883058202, + "val_score": 0.9480603526230034 + }, + { + "epoch": 8, + "test_score": 0.9505410257329664, + "train_loss": 0.183949869193814, + "val_score": 0.949131125987633 + }, + { + "epoch": 9, + "test_score": 0.9476778271773705, + "train_loss": 0.18238650302995335, + "val_score": 0.948899963859157 + }, + { + "epoch": 10, + "test_score": 0.9484377200281172, + "train_loss": 0.1812233101237904, + "val_score": 0.9493756433296019 + }, + { + "epoch": 11, + "test_score": 0.9511458689828213, + "train_loss": 0.18025330684401772, + "val_score": 0.9503379660303507 + }, + { + "epoch": 12, + "test_score": 0.9503413000042816, + "train_loss": 0.1791025774045424, + "val_score": 0.9504818313090763 + }, + { + "epoch": 13, + "test_score": 0.9496615535655901, + "train_loss": 0.17824541519988668, + "val_score": 0.9503709891915616 + }, + { + "epoch": 14, + "test_score": 0.9510270145357611, + "train_loss": 0.17772333513606678, + "val_score": 0.950913761245393 + }, + { + "epoch": 15, + "test_score": 0.950622839345031, + "train_loss": 0.17640978937799282, + "val_score": 0.9509986754336184 + }, + { + "epoch": 16, + "test_score": 0.9512123153600153, + "train_loss": 0.1757170327685096, + "val_score": 0.9515215245685725 + }, + { + "epoch": 17, + "test_score": 0.9499410028657478, + "train_loss": 0.17498180933974006, + "val_score": 0.9509867885023199 + }, + { + "epoch": 18, + "test_score": 0.9513152468388678, + "train_loss": 0.17456490248441697, + "val_score": 0.9513378134520921 + }, + { + "epoch": 19, + "test_score": 0.9511202086000653, + "train_loss": 0.17379635247317227, + "val_score": 0.9516870623958563 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9511202086000653, + "score": 0.9511202086000653 + }, + "val": { + "roc-auc": 0.9516870623958563, + "score": 0.9516870623958563 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0", + "seed": 0, + "task_type": "binclass", + "time_seconds": 58.163018226623535, + "variant": "rla_only" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1.toml new file mode 100644 index 000000000..b4441c2fb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1.toml @@ -0,0 +1,30 @@ +dataset = "homesite-insurance" +variant = "rla_only" +seed = 1 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1/report.json new file mode 100644 index 000000000..0beaadf5d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1/report.json @@ -0,0 +1,179 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 19, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "rla_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.9153453341450584, + "train_loss": 0.40274501984769645, + "val_score": 0.913644257591419 + }, + { + "epoch": 1, + "test_score": 0.9386762760531412, + "train_loss": 0.22882334332574497, + "val_score": 0.9339664964731247 + }, + { + "epoch": 2, + "test_score": 0.9417633747375609, + "train_loss": 0.20528600473295558, + "val_score": 0.9393599453707585 + }, + { + "epoch": 3, + "test_score": 0.9456237884409205, + "train_loss": 0.19763150689276782, + "val_score": 0.9433622452457857 + }, + { + "epoch": 4, + "test_score": 0.9483589387870844, + "train_loss": 0.19308107224377719, + "val_score": 0.9451832756433243 + }, + { + "epoch": 5, + "test_score": 0.9491328820263178, + "train_loss": 0.19030495123429733, + "val_score": 0.9464752936375359 + }, + { + "epoch": 6, + "test_score": 0.9487128442731776, + "train_loss": 0.18752631152218038, + "val_score": 0.9474417222531636 + }, + { + "epoch": 7, + "test_score": 0.9503851666943889, + "train_loss": 0.18584440404718572, + "val_score": 0.9485849338034206 + }, + { + "epoch": 8, + "test_score": 0.9505842158781422, + "train_loss": 0.18417474221099506, + "val_score": 0.9486520615552427 + }, + { + "epoch": 9, + "test_score": 0.9488525024768792, + "train_loss": 0.18312697573141618, + "val_score": 0.9484094608930247 + }, + { + "epoch": 10, + "test_score": 0.950910757537991, + "train_loss": 0.18155857798728076, + "val_score": 0.9492701907748727 + }, + { + "epoch": 11, + "test_score": 0.9498996007321764, + "train_loss": 0.1806687345558947, + "val_score": 0.9495692631528675 + }, + { + "epoch": 12, + "test_score": 0.9505066548342181, + "train_loss": 0.179966039955616, + "val_score": 0.9505498646481203 + }, + { + "epoch": 13, + "test_score": 0.9518296384476232, + "train_loss": 0.17865862548351288, + "val_score": 0.9497052682861931 + }, + { + "epoch": 14, + "test_score": 0.9514363967062314, + "train_loss": 0.1780030677264387, + "val_score": 0.9504259222882722 + }, + { + "epoch": 15, + "test_score": 0.951733279119533, + "train_loss": 0.1766317753629251, + "val_score": 0.9503818826145489 + }, + { + "epoch": 16, + "test_score": 0.951395369088604, + "train_loss": 0.17607535245743663, + "val_score": 0.9512202717475583 + }, + { + "epoch": 17, + "test_score": 0.9506180189623946, + "train_loss": 0.17567047043280168, + "val_score": 0.9505782191994796 + }, + { + "epoch": 18, + "test_score": 0.9511110631623314, + "train_loss": 0.17454163973981685, + "val_score": 0.9500785988163677 + }, + { + "epoch": 19, + "test_score": 0.9523188529200016, + "train_loss": 0.17404835318977183, + "val_score": 0.9514079656893926 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9523188529200016, + "score": 0.9523188529200016 + }, + "val": { + "roc-auc": 0.9514079656893926, + "score": 0.9514079656893926 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1", + "seed": 1, + "task_type": "binclass", + "time_seconds": 58.812034130096436, + "variant": "rla_only" +} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2.toml new file mode 100644 index 000000000..f849c76fb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2.toml @@ -0,0 +1,30 @@ +dataset = "homesite-insurance" +variant = "rla_only" +seed = 2 +batch_size = 2048 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/homesite-insurance" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2/report.json new file mode 100644 index 000000000..fb427d9a5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2/report.json @@ -0,0 +1,179 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 19, + "config": { + "amp": true, + "batch_size": 2048, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/homesite-insurance" + }, + "dataset": "homesite-insurance", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "rla_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2.toml", + "dataset": "homesite-insurance", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": 0.9174934367892956, + "train_loss": 0.39997752498496664, + "val_score": 0.913728908016376 + }, + { + "epoch": 1, + "test_score": 0.9405841690032434, + "train_loss": 0.22608513818545775, + "val_score": 0.9356164412226329 + }, + { + "epoch": 2, + "test_score": 0.9455614254956344, + "train_loss": 0.20442303175275975, + "val_score": 0.9414721352482383 + }, + { + "epoch": 3, + "test_score": 0.9465339346722416, + "train_loss": 0.19598024392669852, + "val_score": 0.9439484766941502 + }, + { + "epoch": 4, + "test_score": 0.9478137643839494, + "train_loss": 0.19159091277555987, + "val_score": 0.9458952254575443 + }, + { + "epoch": 5, + "test_score": 0.949862221624715, + "train_loss": 0.18939599990844727, + "val_score": 0.9463206404411305 + }, + { + "epoch": 6, + "test_score": 0.9460805408377501, + "train_loss": 0.18695615828037263, + "val_score": 0.9467469346356111 + }, + { + "epoch": 7, + "test_score": 0.9487346145225781, + "train_loss": 0.1854052790186622, + "val_score": 0.9475296081742135 + }, + { + "epoch": 8, + "test_score": 0.9508778122160627, + "train_loss": 0.18390980579636312, + "val_score": 0.9494556427289249 + }, + { + "epoch": 9, + "test_score": 0.9500001763849286, + "train_loss": 0.1823697333986109, + "val_score": 0.9494444503742333 + }, + { + "epoch": 10, + "test_score": 0.9504842442833645, + "train_loss": 0.18122355111620642, + "val_score": 0.9495899158167878 + }, + { + "epoch": 11, + "test_score": 0.9515561934837319, + "train_loss": 0.18017809607765892, + "val_score": 0.9499209211344838 + }, + { + "epoch": 12, + "test_score": 0.9521558297538468, + "train_loss": 0.17910988574678247, + "val_score": 0.950046912055717 + }, + { + "epoch": 13, + "test_score": 0.9523240840620607, + "train_loss": 0.17836766649376262, + "val_score": 0.950473672231972 + }, + { + "epoch": 14, + "test_score": 0.9518764891841244, + "train_loss": 0.17786945118145508, + "val_score": 0.9510305468285569 + }, + { + "epoch": 15, + "test_score": 0.9507872035198894, + "train_loss": 0.1764437587423758, + "val_score": 0.9505283063969785 + }, + { + "epoch": 16, + "test_score": 0.9512406215166995, + "train_loss": 0.1760691683400761, + "val_score": 0.9499502252336094 + }, + { + "epoch": 17, + "test_score": 0.9515706546316413, + "train_loss": 0.17495807666670193, + "val_score": 0.9506721365072682 + }, + { + "epoch": 18, + "test_score": 0.95036785439284, + "train_loss": 0.17458564910021696, + "val_score": 0.9506224610917088 + }, + { + "epoch": 19, + "test_score": 0.9524476380801628, + "train_loss": 0.17387819655916908, + "val_score": 0.9513618950385021 + } + ], + "inference_mode": "mean", + "metric_direction": "higher", + "metrics": { + "test": { + "roc-auc": 0.9524476380801628, + "score": 0.9524476380801628 + }, + "val": { + "roc-auc": 0.9513618950385021, + "score": 0.9513618950385021 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2", + "seed": 2, + "task_type": "binclass", + "time_seconds": 60.62201809883118, + "variant": "rla_only" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..4e8978695 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0.toml @@ -0,0 +1,35 @@ +dataset = "sberbank-housing" +variant = "all_four_combined" +seed = 0 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..0d77d461d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0/report.json @@ -0,0 +1,106 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_four_combined" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.529940564415122, + "train_loss": 0.6847074972955804, + "val_score": -11.629337281019204 + }, + { + "epoch": 1, + "test_score": -11.552026902735443, + "train_loss": 0.5718414093318739, + "val_score": -11.661667615617779 + }, + { + "epoch": 2, + "test_score": -11.542241675493882, + "train_loss": 0.5511937800206637, + "val_score": -11.652930392959707 + }, + { + "epoch": 3, + "test_score": -11.57101077716521, + "train_loss": 0.5341950419702028, + "val_score": -11.682380411755172 + }, + { + "epoch": 4, + "test_score": -11.572264141613829, + "train_loss": 0.5156774677728352, + "val_score": -11.690614010077367 + }, + { + "epoch": 5, + "test_score": -11.582711694809559, + "train_loss": 0.515990381178103, + "val_score": -11.692023559546577 + }, + { + "epoch": 6, + "test_score": -11.577212978586967, + "train_loss": 0.5116662116427171, + "val_score": -11.67999320252103 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.529940564415122, + "score": -11.529940564415122 + }, + "val": { + "rmse": 11.629337281019204, + "score": -11.629337281019204 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 6.405361175537109, + "variant": "all_four_combined" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1.toml new file mode 100644 index 000000000..5233af024 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1.toml @@ -0,0 +1,35 @@ +dataset = "sberbank-housing" +variant = "all_four_combined" +seed = 1 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1/report.json new file mode 100644 index 000000000..3a7d4d53d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1/report.json @@ -0,0 +1,106 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_four_combined" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.5397613458401, + "train_loss": 0.6788979737382186, + "val_score": -11.640582398202245 + }, + { + "epoch": 1, + "test_score": -11.555400586515542, + "train_loss": 0.569147073908856, + "val_score": -11.668715451500475 + }, + { + "epoch": 2, + "test_score": -11.541570743387114, + "train_loss": 0.5481514789556202, + "val_score": -11.654005391915891 + }, + { + "epoch": 3, + "test_score": -11.571300230037759, + "train_loss": 0.533915389525263, + "val_score": -11.689820411157616 + }, + { + "epoch": 4, + "test_score": -11.551482028750549, + "train_loss": 0.5247134566307068, + "val_score": -11.675892987295715 + }, + { + "epoch": 5, + "test_score": -11.548386656801894, + "train_loss": 0.516610744752382, + "val_score": -11.66929604101179 + }, + { + "epoch": 6, + "test_score": -11.611992234214632, + "train_loss": 0.5065359049721768, + "val_score": -11.706363245247752 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.5397613458401, + "score": -11.5397613458401 + }, + "val": { + "rmse": 11.640582398202245, + "score": -11.640582398202245 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 6.378625154495239, + "variant": "all_four_combined" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2.toml new file mode 100644 index 000000000..645cf2440 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2.toml @@ -0,0 +1,35 @@ +dataset = "sberbank-housing" +variant = "all_four_combined" +seed = 2 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2/report.json new file mode 100644 index 000000000..5ba1ed4b9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2/report.json @@ -0,0 +1,106 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_four_combined" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.537548960274977, + "train_loss": 0.6803103512839267, + "val_score": -11.640932383735219 + }, + { + "epoch": 1, + "test_score": -11.552889402224842, + "train_loss": 0.571699343229595, + "val_score": -11.667362592013484 + }, + { + "epoch": 2, + "test_score": -11.533435810787802, + "train_loss": 0.5532804304047635, + "val_score": -11.653257093143798 + }, + { + "epoch": 3, + "test_score": -11.540510392827223, + "train_loss": 0.5380085847879711, + "val_score": -11.664698262144418 + }, + { + "epoch": 4, + "test_score": -11.55832180104211, + "train_loss": 0.5288247748425132, + "val_score": -11.67843129218514 + }, + { + "epoch": 5, + "test_score": -11.562065157574914, + "train_loss": 0.5185170534409975, + "val_score": -11.677382062391155 + }, + { + "epoch": 6, + "test_score": -11.559098686799585, + "train_loss": 0.5019456665766867, + "val_score": -11.66983214531403 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.537548960274977, + "score": -11.537548960274977 + }, + "val": { + "rmse": 11.640932383735219, + "score": -11.640932383735219 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 6.827687740325928, + "variant": "all_four_combined" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..1ee5d2edc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml @@ -0,0 +1,34 @@ +dataset = "sberbank-housing" +variant = "all_minus_cf_fisd" +seed = 0 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..280a56631 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0/report.json @@ -0,0 +1,117 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 2, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_minus_cf_fisd" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.513746587172779, + "train_loss": 0.7153874667067277, + "val_score": -11.650224125224748 + }, + { + "epoch": 1, + "test_score": -11.52669511687469, + "train_loss": 0.563560238010005, + "val_score": -11.652123751275518 + }, + { + "epoch": 2, + "test_score": -11.495833595850131, + "train_loss": 0.537884428312904, + "val_score": -11.627223960865845 + }, + { + "epoch": 3, + "test_score": -11.543432731791633, + "train_loss": 0.5185733155200356, + "val_score": -11.665168520694888 + }, + { + "epoch": 4, + "test_score": -11.594056652363017, + "train_loss": 0.5044406621079696, + "val_score": -11.705019299712436 + }, + { + "epoch": 5, + "test_score": -11.58571953288731, + "train_loss": 0.5075031064058605, + "val_score": -11.69188522239724 + }, + { + "epoch": 6, + "test_score": -11.565673392647339, + "train_loss": 0.5029034708675585, + "val_score": -11.668180603617378 + }, + { + "epoch": 7, + "test_score": -11.585750483150088, + "train_loss": 0.4975504420305553, + "val_score": -11.693598008098277 + }, + { + "epoch": 8, + "test_score": -11.576650837115723, + "train_loss": 0.48497160328061956, + "val_score": -11.689399442493693 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.495833595850131, + "score": -11.495833595850131 + }, + "val": { + "rmse": 11.627223960865845, + "score": -11.627223960865845 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 8.408822298049927, + "variant": "all_minus_cf_fisd" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1.toml new file mode 100644 index 000000000..afca58636 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1.toml @@ -0,0 +1,34 @@ +dataset = "sberbank-housing" +variant = "all_minus_cf_fisd" +seed = 1 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1/report.json new file mode 100644 index 000000000..e873837eb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1/report.json @@ -0,0 +1,105 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_minus_cf_fisd" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.52660642338622, + "train_loss": 0.7090223055136832, + "val_score": -11.641336754666401 + }, + { + "epoch": 1, + "test_score": -11.571683958219143, + "train_loss": 0.5640872654161955, + "val_score": -11.688843349018354 + }, + { + "epoch": 2, + "test_score": -11.584105395011493, + "train_loss": 0.5382686147564336, + "val_score": -11.7023198623663 + }, + { + "epoch": 3, + "test_score": -11.580835599119268, + "train_loss": 0.526674791386253, + "val_score": -11.693449250374918 + }, + { + "epoch": 4, + "test_score": -11.584639514489178, + "train_loss": 0.5151608900019997, + "val_score": -11.702422218968842 + }, + { + "epoch": 5, + "test_score": -11.582740677050353, + "train_loss": 0.5098892026825955, + "val_score": -11.69189435791984 + }, + { + "epoch": 6, + "test_score": -11.622656144972105, + "train_loss": 0.49971666304688706, + "val_score": -11.708356719259525 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.52660642338622, + "score": -11.52660642338622 + }, + "val": { + "rmse": 11.641336754666401, + "score": -11.641336754666401 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 6.753610610961914, + "variant": "all_minus_cf_fisd" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2.toml new file mode 100644 index 000000000..1ee1df5df --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2.toml @@ -0,0 +1,34 @@ +dataset = "sberbank-housing" +variant = "all_minus_cf_fisd" +seed = 2 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2/report.json new file mode 100644 index 000000000..745a80212 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2/report.json @@ -0,0 +1,105 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_minus_cf_fisd" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.512635297114862, + "train_loss": 0.7103617254056429, + "val_score": -11.642066159740436 + }, + { + "epoch": 1, + "test_score": -11.561014605812948, + "train_loss": 0.5649806794367338, + "val_score": -11.6840142748627 + }, + { + "epoch": 2, + "test_score": -11.550686136317902, + "train_loss": 0.5418291295829573, + "val_score": -11.675974012502126 + }, + { + "epoch": 3, + "test_score": -11.552818080053902, + "train_loss": 0.523169795149251, + "val_score": -11.678895771660628 + }, + { + "epoch": 4, + "test_score": -11.597623357986365, + "train_loss": 0.5167672179247204, + "val_score": -11.710011064506274 + }, + { + "epoch": 5, + "test_score": -11.571608136649118, + "train_loss": 0.5108516859380823, + "val_score": -11.677881209615828 + }, + { + "epoch": 6, + "test_score": -11.561509538447702, + "train_loss": 0.4943600296974182, + "val_score": -11.662550790559619 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.512635297114862, + "score": -11.512635297114862 + }, + "val": { + "rmse": 11.642066159740436, + "score": -11.642066159740436 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 7.154858112335205, + "variant": "all_minus_cf_fisd" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0.toml new file mode 100644 index 000000000..35a1f75ee --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0.toml @@ -0,0 +1,33 @@ +dataset = "sberbank-housing" +variant = "all_minus_esam" +seed = 0 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0/report.json new file mode 100644 index 000000000..213ca4f77 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0/report.json @@ -0,0 +1,104 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_minus_esam" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.529550814849099, + "train_loss": 0.6817392173566317, + "val_score": -11.629953949067119 + }, + { + "epoch": 1, + "test_score": -11.550435138118512, + "train_loss": 0.5682083619268317, + "val_score": -11.660886440704575 + }, + { + "epoch": 2, + "test_score": -11.530798761072859, + "train_loss": 0.5492272831891712, + "val_score": -11.642030771869832 + }, + { + "epoch": 3, + "test_score": -11.537696421760417, + "train_loss": 0.534926375276164, + "val_score": -11.651297409440813 + }, + { + "epoch": 4, + "test_score": -11.564843512505908, + "train_loss": 0.5191069511990798, + "val_score": -11.687417096338654 + }, + { + "epoch": 5, + "test_score": -11.562500659839507, + "train_loss": 0.5150835153303648, + "val_score": -11.682755920101723 + }, + { + "epoch": 6, + "test_score": -11.584154131965139, + "train_loss": 0.5153278159467798, + "val_score": -11.693979680442478 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.529550814849099, + "score": -11.529550814849099 + }, + "val": { + "rmse": 11.629953949067119, + "score": -11.629953949067119 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 5.24409818649292, + "variant": "all_minus_esam" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1.toml new file mode 100644 index 000000000..5e5dbfdac --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1.toml @@ -0,0 +1,33 @@ +dataset = "sberbank-housing" +variant = "all_minus_esam" +seed = 1 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1/report.json new file mode 100644 index 000000000..88e5e3b87 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1/report.json @@ -0,0 +1,104 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_minus_esam" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.535907580086148, + "train_loss": 0.6764907836914062, + "val_score": -11.636771200368536 + }, + { + "epoch": 1, + "test_score": -11.55272892672103, + "train_loss": 0.5704263276175449, + "val_score": -11.669556904596222 + }, + { + "epoch": 2, + "test_score": -11.540304789889854, + "train_loss": 0.5479874469731983, + "val_score": -11.660911957294818 + }, + { + "epoch": 3, + "test_score": -11.566767714294487, + "train_loss": 0.5281973810572373, + "val_score": -11.694827144237479 + }, + { + "epoch": 4, + "test_score": -11.54475385414234, + "train_loss": 0.5193592780514767, + "val_score": -11.669544482653006 + }, + { + "epoch": 5, + "test_score": -11.54118006445933, + "train_loss": 0.5155376776268608, + "val_score": -11.653920940654078 + }, + { + "epoch": 6, + "test_score": -11.589382284586234, + "train_loss": 0.5071287123780501, + "val_score": -11.688474563520696 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.535907580086148, + "score": -11.535907580086148 + }, + "val": { + "rmse": 11.636771200368536, + "score": -11.636771200368536 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 5.403448104858398, + "variant": "all_minus_esam" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2.toml new file mode 100644 index 000000000..fcafe5ea2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2.toml @@ -0,0 +1,33 @@ +dataset = "sberbank-housing" +variant = "all_minus_esam" +seed = 2 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2/report.json new file mode 100644 index 000000000..6f5cfb23c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2/report.json @@ -0,0 +1,104 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_minus_esam" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.53778767500918, + "train_loss": 0.6787219533794805, + "val_score": -11.640474254476947 + }, + { + "epoch": 1, + "test_score": -11.552421177337404, + "train_loss": 0.5701834920205568, + "val_score": -11.668965866960828 + }, + { + "epoch": 2, + "test_score": -11.534591397446842, + "train_loss": 0.5533530414104462, + "val_score": -11.655044943364052 + }, + { + "epoch": 3, + "test_score": -11.539088285981682, + "train_loss": 0.5404668748378754, + "val_score": -11.660349270806064 + }, + { + "epoch": 4, + "test_score": -11.551949631261518, + "train_loss": 0.5275069977107801, + "val_score": -11.672618837158101 + }, + { + "epoch": 5, + "test_score": -11.547939390852582, + "train_loss": 0.5141889217652773, + "val_score": -11.66554000500613 + }, + { + "epoch": 6, + "test_score": -11.560263587560069, + "train_loss": 0.5009887877263521, + "val_score": -11.672157376102328 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.53778767500918, + "score": -11.53778767500918 + }, + "val": { + "rmse": 11.640474254476947, + "score": -11.640474254476947 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 5.441632270812988, + "variant": "all_minus_esam" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0.toml new file mode 100644 index 000000000..67db20514 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0.toml @@ -0,0 +1,33 @@ +dataset = "sberbank-housing" +variant = "all_minus_mfb" +seed = 0 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0/report.json new file mode 100644 index 000000000..060d2806a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0/report.json @@ -0,0 +1,104 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_minus_mfb" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.529854542723736, + "train_loss": 0.6731284982279727, + "val_score": -11.628387941925986 + }, + { + "epoch": 1, + "test_score": -11.55628990220092, + "train_loss": 0.5665753781795502, + "val_score": -11.665185525516796 + }, + { + "epoch": 2, + "test_score": -11.533286971805715, + "train_loss": 0.5451070242806485, + "val_score": -11.645597852442798 + }, + { + "epoch": 3, + "test_score": -11.543267498249048, + "train_loss": 0.5240256049131092, + "val_score": -11.662766667595363 + }, + { + "epoch": 4, + "test_score": -11.569713754764527, + "train_loss": 0.5067465054361444, + "val_score": -11.69366259967839 + }, + { + "epoch": 5, + "test_score": -11.55628462064683, + "train_loss": 0.5012774718435187, + "val_score": -11.671706355458351 + }, + { + "epoch": 6, + "test_score": -11.601807788863189, + "train_loss": 0.49037294011366994, + "val_score": -11.707959224566062 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.529854542723736, + "score": -11.529854542723736 + }, + "val": { + "rmse": 11.628387941925986, + "score": -11.628387941925986 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 6.601038455963135, + "variant": "all_minus_mfb" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1.toml new file mode 100644 index 000000000..75c9b75bd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1.toml @@ -0,0 +1,33 @@ +dataset = "sberbank-housing" +variant = "all_minus_mfb" +seed = 1 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1/report.json new file mode 100644 index 000000000..f6828bf47 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1/report.json @@ -0,0 +1,104 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_minus_mfb" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.527989037555509, + "train_loss": 0.6697249318424024, + "val_score": -11.635427085077215 + }, + { + "epoch": 1, + "test_score": -11.541932985467193, + "train_loss": 0.5654332637786865, + "val_score": -11.66612860029216 + }, + { + "epoch": 2, + "test_score": -11.535246200546677, + "train_loss": 0.5421643006174188, + "val_score": -11.660955793357955 + }, + { + "epoch": 3, + "test_score": -11.5517871613484, + "train_loss": 0.5226815766409824, + "val_score": -11.682632493472482 + }, + { + "epoch": 4, + "test_score": -11.538862160794725, + "train_loss": 0.5097503881705435, + "val_score": -11.677146854732632 + }, + { + "epoch": 5, + "test_score": -11.533347169136466, + "train_loss": 0.5043469306669737, + "val_score": -11.656677401891065 + }, + { + "epoch": 6, + "test_score": -11.604282088650239, + "train_loss": 0.4942637085914612, + "val_score": -11.70735774382063 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.527989037555509, + "score": -11.527989037555509 + }, + "val": { + "rmse": 11.635427085077215, + "score": -11.635427085077215 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 6.853307008743286, + "variant": "all_minus_mfb" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2.toml new file mode 100644 index 000000000..039ada680 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2.toml @@ -0,0 +1,33 @@ +dataset = "sberbank-housing" +variant = "all_minus_mfb" +seed = 2 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 +esam_enabled = true +esam_rho = 0.0025 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2/report.json new file mode 100644 index 000000000..e9bbf8124 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2/report.json @@ -0,0 +1,104 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_minus_mfb" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.53653783950218, + "train_loss": 0.6675687244063929, + "val_score": -11.638006993773004 + }, + { + "epoch": 1, + "test_score": -11.563427037372303, + "train_loss": 0.5652898048099718, + "val_score": -11.677691091918867 + }, + { + "epoch": 2, + "test_score": -11.543884137773045, + "train_loss": 0.5458332911918038, + "val_score": -11.662630600111127 + }, + { + "epoch": 3, + "test_score": -11.554151335599578, + "train_loss": 0.5292629395660601, + "val_score": -11.67665486339423 + }, + { + "epoch": 4, + "test_score": -11.5687773289453, + "train_loss": 0.5131576892576719, + "val_score": -11.6912705158342 + }, + { + "epoch": 5, + "test_score": -11.555286363579915, + "train_loss": 0.502765333966205, + "val_score": -11.674081535071613 + }, + { + "epoch": 6, + "test_score": -11.562772510513334, + "train_loss": 0.4887320367913497, + "val_score": -11.675086625163777 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.53653783950218, + "score": -11.53653783950218 + }, + "val": { + "rmse": 11.638006993773004, + "score": -11.638006993773004 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 7.148878574371338, + "variant": "all_minus_mfb" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0.toml new file mode 100644 index 000000000..dbdd34242 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0.toml @@ -0,0 +1,32 @@ +dataset = "sberbank-housing" +variant = "all_minus_rla" +seed = 0 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0/report.json new file mode 100644 index 000000000..fa2c9ee04 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0/report.json @@ -0,0 +1,127 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 4, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "all_minus_rla" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.566356778642936, + "train_loss": 0.6788321796216463, + "val_score": -11.664222097869944 + }, + { + "epoch": 1, + "test_score": -11.559857700620443, + "train_loss": 0.5704584325614729, + "val_score": -11.670672864172788 + }, + { + "epoch": 2, + "test_score": -11.561442228854185, + "train_loss": 0.5435581897434435, + "val_score": -11.671773029173043 + }, + { + "epoch": 3, + "test_score": -11.562635926143608, + "train_loss": 0.5292612361280542, + "val_score": -11.67968292746566 + }, + { + "epoch": 4, + "test_score": -11.530414333922007, + "train_loss": 0.516992837190628, + "val_score": -11.648797073801292 + }, + { + "epoch": 5, + "test_score": -11.546760692727513, + "train_loss": 0.5125646606871956, + "val_score": -11.659984818672779 + }, + { + "epoch": 6, + "test_score": -11.536368539200957, + "train_loss": 0.5112680792808533, + "val_score": -11.652399403780938 + }, + { + "epoch": 7, + "test_score": -11.557969974086063, + "train_loss": 0.5115332070149874, + "val_score": -11.681961787352837 + }, + { + "epoch": 8, + "test_score": -11.566561258881947, + "train_loss": 0.4950885882503108, + "val_score": -11.689068530901732 + }, + { + "epoch": 9, + "test_score": -11.561465985226054, + "train_loss": 0.4844362704377425, + "val_score": -11.683070031725862 + }, + { + "epoch": 10, + "test_score": -11.546094648656844, + "train_loss": 0.4920222649448796, + "val_score": -11.659160997097947 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.530414333922007, + "score": -11.530414333922007 + }, + "val": { + "rmse": 11.648797073801292, + "score": -11.648797073801292 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 9.661595106124878, + "variant": "all_minus_rla" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1.toml new file mode 100644 index 000000000..9be092dff --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1.toml @@ -0,0 +1,32 @@ +dataset = "sberbank-housing" +variant = "all_minus_rla" +seed = 1 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1/report.json new file mode 100644 index 000000000..916dc5a03 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1/report.json @@ -0,0 +1,133 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 5, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "all_minus_rla" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.56395749365411, + "train_loss": 0.6744678585152877, + "val_score": -11.668505569129103 + }, + { + "epoch": 1, + "test_score": -11.537794287535394, + "train_loss": 0.5657629998106706, + "val_score": -11.657687264774315 + }, + { + "epoch": 2, + "test_score": -11.548418367762531, + "train_loss": 0.5490596780651494, + "val_score": -11.67048524398528 + }, + { + "epoch": 3, + "test_score": -11.549676165644213, + "train_loss": 0.536047350419195, + "val_score": -11.674848103811472 + }, + { + "epoch": 4, + "test_score": -11.556204736846864, + "train_loss": 0.52565745617214, + "val_score": -11.684645686721474 + }, + { + "epoch": 5, + "test_score": -11.537306272919873, + "train_loss": 0.5129967410313455, + "val_score": -11.650615731532309 + }, + { + "epoch": 6, + "test_score": -11.59410995377037, + "train_loss": 0.49969515361283956, + "val_score": -11.707428776200379 + }, + { + "epoch": 7, + "test_score": -11.594070471269934, + "train_loss": 0.4978504071110173, + "val_score": -11.699758047878865 + }, + { + "epoch": 8, + "test_score": -11.604694968524592, + "train_loss": 0.49357654075873525, + "val_score": -11.712956901440387 + }, + { + "epoch": 9, + "test_score": -11.56541084559229, + "train_loss": 0.48850819939061213, + "val_score": -11.66822833553725 + }, + { + "epoch": 10, + "test_score": -11.583416469650697, + "train_loss": 0.49180947636303146, + "val_score": -11.685402421857033 + }, + { + "epoch": 11, + "test_score": -11.57456217322941, + "train_loss": 0.49050161556193705, + "val_score": -11.673448898234296 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.537306272919873, + "score": -11.537306272919873 + }, + "val": { + "rmse": 11.650615731532309, + "score": -11.650615731532309 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 10.637221097946167, + "variant": "all_minus_rla" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2.toml new file mode 100644 index 000000000..e24f8e419 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2.toml @@ -0,0 +1,32 @@ +dataset = "sberbank-housing" +variant = "all_minus_rla" +seed = 2 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 +mfb_enabled = true +mfb_keep = 0.8 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2/report.json new file mode 100644 index 000000000..bc7bfb855 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2/report.json @@ -0,0 +1,103 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "all_minus_rla" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.537770482423289, + "train_loss": 0.6824794066579718, + "val_score": -11.64186693773376 + }, + { + "epoch": 1, + "test_score": -11.56087866060679, + "train_loss": 0.5675985232779854, + "val_score": -11.674113558132115 + }, + { + "epoch": 2, + "test_score": -11.552444952260107, + "train_loss": 0.562320916276229, + "val_score": -11.668439530592837 + }, + { + "epoch": 3, + "test_score": -11.552125967971559, + "train_loss": 0.5406928344776756, + "val_score": -11.67393056803177 + }, + { + "epoch": 4, + "test_score": -11.531967843721764, + "train_loss": 0.5256602340622952, + "val_score": -11.65659100644788 + }, + { + "epoch": 5, + "test_score": -11.576962555511646, + "train_loss": 0.5158363957154123, + "val_score": -11.694865634206682 + }, + { + "epoch": 6, + "test_score": -11.559399658242826, + "train_loss": 0.5086805240104073, + "val_score": -11.677255312186018 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.537770482423289, + "score": -11.537770482423289 + }, + "val": { + "rmse": 11.64186693773376, + "score": -11.64186693773376 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 6.342571973800659, + "variant": "all_minus_rla" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..54fe49077 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0.toml @@ -0,0 +1,27 @@ +dataset = "sberbank-housing" +variant = "baseline_plr" +seed = 0 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..18b204a91 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0/report.json @@ -0,0 +1,128 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 5, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "baseline_plr" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.591843440565695, + "train_loss": 0.693355569713994, + "val_score": -11.707670543731014 + }, + { + "epoch": 1, + "test_score": -11.569860146879883, + "train_loss": 0.5572547583203566, + "val_score": -11.699261138278803 + }, + { + "epoch": 2, + "test_score": -11.557012792142023, + "train_loss": 0.5240200240361063, + "val_score": -11.679103507557178 + }, + { + "epoch": 3, + "test_score": -11.546682725268232, + "train_loss": 0.5031786121820149, + "val_score": -11.66853172290153 + }, + { + "epoch": 4, + "test_score": -11.576095918058972, + "train_loss": 0.4946268655751881, + "val_score": -11.689467973309283 + }, + { + "epoch": 5, + "test_score": -11.546495072614636, + "train_loss": 0.49316220534475225, + "val_score": -11.653776913834497 + }, + { + "epoch": 6, + "test_score": -11.576089327411522, + "train_loss": 0.4814813748786324, + "val_score": -11.692044440483546 + }, + { + "epoch": 7, + "test_score": -11.570189192666712, + "train_loss": 0.47625335894132914, + "val_score": -11.6955636506517 + }, + { + "epoch": 8, + "test_score": -11.549253391724072, + "train_loss": 0.4664542518163982, + "val_score": -11.665307174673636 + }, + { + "epoch": 9, + "test_score": -11.559674221847015, + "train_loss": 0.46214914792462397, + "val_score": -11.676656823560215 + }, + { + "epoch": 10, + "test_score": -11.541870849859816, + "train_loss": 0.46384631332598236, + "val_score": -11.672688120104073 + }, + { + "epoch": 11, + "test_score": -11.534430667367701, + "train_loss": 0.45383137621377645, + "val_score": -11.660644357455324 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.546495072614636, + "score": -11.546495072614636 + }, + "val": { + "rmse": 11.653776913834497, + "score": -11.653776913834497 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 8.12721848487854, + "variant": "baseline_plr" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1.toml new file mode 100644 index 000000000..96b89a263 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1.toml @@ -0,0 +1,27 @@ +dataset = "sberbank-housing" +variant = "baseline_plr" +seed = 1 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1/report.json new file mode 100644 index 000000000..8fd14007c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1/report.json @@ -0,0 +1,104 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 1, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "baseline_plr" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.5648421930942, + "train_loss": 0.689470742878161, + "val_score": -11.693471433664252 + }, + { + "epoch": 1, + "test_score": -11.510137982108967, + "train_loss": 0.5526794289287768, + "val_score": -11.654311113459125 + }, + { + "epoch": 2, + "test_score": -11.529714259819535, + "train_loss": 0.5296151810570767, + "val_score": -11.669904059871339 + }, + { + "epoch": 3, + "test_score": -11.544838443119694, + "train_loss": 0.5117689402479875, + "val_score": -11.687559403049239 + }, + { + "epoch": 4, + "test_score": -11.559208251872455, + "train_loss": 0.5005878413978376, + "val_score": -11.688715417668204 + }, + { + "epoch": 5, + "test_score": -11.55868681844079, + "train_loss": 0.48899377960907786, + "val_score": -11.673540397418268 + }, + { + "epoch": 6, + "test_score": -11.59862849579201, + "train_loss": 0.47632234974911336, + "val_score": -11.72544405163437 + }, + { + "epoch": 7, + "test_score": -11.569045075332628, + "train_loss": 0.47122232223811905, + "val_score": -11.684690739624394 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.510137982108967, + "score": -11.510137982108967 + }, + "val": { + "rmse": 11.654311113459125, + "score": -11.654311113459125 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 5.4021241664886475, + "variant": "baseline_plr" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2.toml new file mode 100644 index 000000000..eb19da287 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2.toml @@ -0,0 +1,27 @@ +dataset = "sberbank-housing" +variant = "baseline_plr" +seed = 2 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2/report.json new file mode 100644 index 000000000..df5e73c37 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2/report.json @@ -0,0 +1,176 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 18, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "baseline_plr" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.554990567146126, + "train_loss": 0.6944742610580042, + "val_score": -11.681583641026789 + }, + { + "epoch": 1, + "test_score": -11.558377247465431, + "train_loss": 0.5524072521611264, + "val_score": -11.694212592316632 + }, + { + "epoch": 2, + "test_score": -11.554818896100436, + "train_loss": 0.5426086312846133, + "val_score": -11.695455362980141 + }, + { + "epoch": 3, + "test_score": -11.546551897331664, + "train_loss": 0.5175957852288297, + "val_score": -11.6712468193992 + }, + { + "epoch": 4, + "test_score": -11.541549590213585, + "train_loss": 0.500424231353559, + "val_score": -11.670191059918428 + }, + { + "epoch": 5, + "test_score": -11.548839851010875, + "train_loss": 0.4906174760115774, + "val_score": -11.668238797301818 + }, + { + "epoch": 6, + "test_score": -11.580356645379265, + "train_loss": 0.4788107119108501, + "val_score": -11.70130407124665 + }, + { + "epoch": 7, + "test_score": -11.558303978920982, + "train_loss": 0.47477054439092936, + "val_score": -11.685612000748238 + }, + { + "epoch": 8, + "test_score": -11.548932998054612, + "train_loss": 0.46741848870327596, + "val_score": -11.67610796423118 + }, + { + "epoch": 9, + "test_score": -11.55564223362945, + "train_loss": 0.4634051119026385, + "val_score": -11.686815864000874 + }, + { + "epoch": 10, + "test_score": -11.528746790674022, + "train_loss": 0.45700703482878835, + "val_score": -11.662469017911688 + }, + { + "epoch": 11, + "test_score": -11.520487572630458, + "train_loss": 0.45120432502345037, + "val_score": -11.655515591391566 + }, + { + "epoch": 12, + "test_score": -11.500814657795715, + "train_loss": 0.454963019019679, + "val_score": -11.632714121897644 + }, + { + "epoch": 13, + "test_score": -11.525585075306372, + "train_loss": 0.4519016460368508, + "val_score": -11.661778833921053 + }, + { + "epoch": 14, + "test_score": -11.52689632942267, + "train_loss": 0.44427210406253215, + "val_score": -11.68127928638818 + }, + { + "epoch": 15, + "test_score": -11.49810377326885, + "train_loss": 0.44250490634064926, + "val_score": -11.632058902512634 + }, + { + "epoch": 16, + "test_score": -11.530547329962946, + "train_loss": 0.43535852589105306, + "val_score": -11.6786279306174 + }, + { + "epoch": 17, + "test_score": -11.479069113388983, + "train_loss": 0.4340797850960179, + "val_score": -11.643101535007421 + }, + { + "epoch": 18, + "test_score": -11.460434895463788, + "train_loss": 0.43446698314265203, + "val_score": -11.624637064857055 + }, + { + "epoch": 19, + "test_score": -11.504066080171528, + "train_loss": 0.4255804733226174, + "val_score": -11.66930388661818 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.460434895463788, + "score": -11.460434895463788 + }, + "val": { + "rmse": 11.624637064857055, + "score": -11.624637064857055 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 14.094439268112183, + "variant": "baseline_plr" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..d24995252 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0.toml @@ -0,0 +1,28 @@ +dataset = "sberbank-housing" +variant = "cf_fisd_only" +seed = 0 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..ce44d72ea --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,129 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 5, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "cf_fisd_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.567909418746003, + "train_loss": 0.6687586307525635, + "val_score": -11.665182909391962 + }, + { + "epoch": 1, + "test_score": -11.56754864967921, + "train_loss": 0.5649684698958146, + "val_score": -11.678224850932839 + }, + { + "epoch": 2, + "test_score": -11.555461989129928, + "train_loss": 0.5369522273540497, + "val_score": -11.666910078143415 + }, + { + "epoch": 3, + "test_score": -11.552720341548957, + "train_loss": 0.5171568503505305, + "val_score": -11.671635759345309 + }, + { + "epoch": 4, + "test_score": -11.56288138095683, + "train_loss": 0.5048333422133797, + "val_score": -11.681953950247498 + }, + { + "epoch": 5, + "test_score": -11.541630897512452, + "train_loss": 0.5006356631454668, + "val_score": -11.652210179841218 + }, + { + "epoch": 6, + "test_score": -11.572904287492442, + "train_loss": 0.4888252600243217, + "val_score": -11.679139436346876 + }, + { + "epoch": 7, + "test_score": -11.592184367207084, + "train_loss": 0.48512670554612813, + "val_score": -11.697678320956939 + }, + { + "epoch": 8, + "test_score": -11.589911552927722, + "train_loss": 0.473800927400589, + "val_score": -11.688119474099343 + }, + { + "epoch": 9, + "test_score": -11.600638510132683, + "train_loss": 0.46804546839312505, + "val_score": -11.697391995320475 + }, + { + "epoch": 10, + "test_score": -11.593598644985759, + "train_loss": 0.4702984408328408, + "val_score": -11.684281992433721 + }, + { + "epoch": 11, + "test_score": -11.600327427945595, + "train_loss": 0.4600174160380113, + "val_score": -11.683457272063137 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.541630897512452, + "score": -11.541630897512452 + }, + "val": { + "rmse": 11.652210179841218, + "score": -11.652210179841218 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 7.491765737533569, + "variant": "cf_fisd_only" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1.toml new file mode 100644 index 000000000..04d71bf4b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1.toml @@ -0,0 +1,28 @@ +dataset = "sberbank-housing" +variant = "cf_fisd_only" +seed = 1 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1/report.json new file mode 100644 index 000000000..17c094d35 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1/report.json @@ -0,0 +1,105 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 1, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "cf_fisd_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.552516937144382, + "train_loss": 0.6644222955954703, + "val_score": -11.662240051447155 + }, + { + "epoch": 1, + "test_score": -11.531481567373374, + "train_loss": 0.5625829837824169, + "val_score": -11.653860711473339 + }, + { + "epoch": 2, + "test_score": -11.545619540172206, + "train_loss": 0.5438563996239713, + "val_score": -11.6724397453735 + }, + { + "epoch": 3, + "test_score": -11.532994580301242, + "train_loss": 0.5255994828123796, + "val_score": -11.674658590221908 + }, + { + "epoch": 4, + "test_score": -11.526772557646096, + "train_loss": 0.5107194577392778, + "val_score": -11.666102441132265 + }, + { + "epoch": 5, + "test_score": -11.546941733780434, + "train_loss": 0.4957810671705949, + "val_score": -11.657945770373138 + }, + { + "epoch": 6, + "test_score": -11.59458044321988, + "train_loss": 0.4840890231885408, + "val_score": -11.704705777126632 + }, + { + "epoch": 7, + "test_score": -11.591498554764918, + "train_loss": 0.47806531661435175, + "val_score": -11.689960730676798 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.531481567373374, + "score": -11.531481567373374 + }, + "val": { + "rmse": 11.653860711473339, + "score": -11.653860711473339 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 5.391960620880127, + "variant": "cf_fisd_only" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2.toml new file mode 100644 index 000000000..7bb90cee0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2.toml @@ -0,0 +1,28 @@ +dataset = "sberbank-housing" +variant = "cf_fisd_only" +seed = 2 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2/report.json new file mode 100644 index 000000000..f7c307400 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2/report.json @@ -0,0 +1,99 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "cf_fisd_enabled": true, + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "cf_fisd_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.533345184614086, + "train_loss": 0.6680415868759155, + "val_score": -11.634575294575916 + }, + { + "epoch": 1, + "test_score": -11.570453609453505, + "train_loss": 0.5608573979453036, + "val_score": -11.68285126464747 + }, + { + "epoch": 2, + "test_score": -11.558421472398113, + "train_loss": 0.5553898183923018, + "val_score": -11.674624608149546 + }, + { + "epoch": 3, + "test_score": -11.556772494019064, + "train_loss": 0.5318876630381534, + "val_score": -11.678057604710867 + }, + { + "epoch": 4, + "test_score": -11.528811644085643, + "train_loss": 0.512807066503324, + "val_score": -11.649647824444898 + }, + { + "epoch": 5, + "test_score": -11.562042062299835, + "train_loss": 0.5008556623207895, + "val_score": -11.673831229347504 + }, + { + "epoch": 6, + "test_score": -11.598899499715143, + "train_loss": 0.48822733289317083, + "val_score": -11.710128990394585 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.533345184614086, + "score": -11.533345184614086 + }, + "val": { + "rmse": 11.634575294575916, + "score": -11.634575294575916 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 5.544011116027832, + "variant": "cf_fisd_only" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0.toml new file mode 100644 index 000000000..ab2fd5f22 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0.toml @@ -0,0 +1,29 @@ +dataset = "sberbank-housing" +variant = "esam_only" +seed = 0 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0/report.json new file mode 100644 index 000000000..a205243b3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0/report.json @@ -0,0 +1,130 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 5, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "esam_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.591146418407833, + "train_loss": 0.6944636043749357, + "val_score": -11.706608293368964 + }, + { + "epoch": 1, + "test_score": -11.570934291825452, + "train_loss": 0.5573361547369706, + "val_score": -11.699037456859099 + }, + { + "epoch": 2, + "test_score": -11.554540255358294, + "train_loss": 0.524962704432638, + "val_score": -11.675947875400203 + }, + { + "epoch": 3, + "test_score": -11.547431983986945, + "train_loss": 0.5038708526837198, + "val_score": -11.668148564273972 + }, + { + "epoch": 4, + "test_score": -11.572368307825029, + "train_loss": 0.4954853465682582, + "val_score": -11.68653384208455 + }, + { + "epoch": 5, + "test_score": -11.547765632938749, + "train_loss": 0.49385242399416474, + "val_score": -11.655595449115308 + }, + { + "epoch": 6, + "test_score": -11.570127868209529, + "train_loss": 0.48245343722795186, + "val_score": -11.685794155049098 + }, + { + "epoch": 7, + "test_score": -11.57227996438985, + "train_loss": 0.4776828602740639, + "val_score": -11.697226327628513 + }, + { + "epoch": 8, + "test_score": -11.553689104939329, + "train_loss": 0.46720715102396515, + "val_score": -11.666946698415652 + }, + { + "epoch": 9, + "test_score": -11.565484728783257, + "train_loss": 0.46342792165906804, + "val_score": -11.678648182228018 + }, + { + "epoch": 10, + "test_score": -11.55179376586097, + "train_loss": 0.4648975808369486, + "val_score": -11.678585467448864 + }, + { + "epoch": 11, + "test_score": -11.540711364505574, + "train_loss": 0.45520147994944926, + "val_score": -11.660473587599665 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.547765632938749, + "score": -11.547765632938749 + }, + "val": { + "rmse": 11.655595449115308, + "score": -11.655595449115308 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 9.331769227981567, + "variant": "esam_only" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1.toml new file mode 100644 index 000000000..673db2d27 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1.toml @@ -0,0 +1,29 @@ +dataset = "sberbank-housing" +variant = "esam_only" +seed = 1 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1/report.json new file mode 100644 index 000000000..e251d10ea --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1/report.json @@ -0,0 +1,106 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 1, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "esam_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.566685924092585, + "train_loss": 0.6905955389926308, + "val_score": -11.69451269601541 + }, + { + "epoch": 1, + "test_score": -11.510244699059466, + "train_loss": 0.5533220720918555, + "val_score": -11.65383845278429 + }, + { + "epoch": 2, + "test_score": -11.530006734510243, + "train_loss": 0.5304650350620872, + "val_score": -11.670209364921169 + }, + { + "epoch": 3, + "test_score": -11.544808704877989, + "train_loss": 0.5126697138736123, + "val_score": -11.687711499590339 + }, + { + "epoch": 4, + "test_score": -11.562260475770744, + "train_loss": 0.5019068388562453, + "val_score": -11.690407783960037 + }, + { + "epoch": 5, + "test_score": -11.558812888679695, + "train_loss": 0.49016095619452626, + "val_score": -11.672905116858763 + }, + { + "epoch": 6, + "test_score": -11.592663490404515, + "train_loss": 0.4767398347980098, + "val_score": -11.721967657216767 + }, + { + "epoch": 7, + "test_score": -11.5657268248608, + "train_loss": 0.4714427041380029, + "val_score": -11.684485061914769 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.510244699059466, + "score": -11.510244699059466 + }, + "val": { + "rmse": 11.65383845278429, + "score": -11.65383845278429 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 6.739201784133911, + "variant": "esam_only" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2.toml new file mode 100644 index 000000000..ba0118965 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2.toml @@ -0,0 +1,29 @@ +dataset = "sberbank-housing" +variant = "esam_only" +seed = 2 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +esam_enabled = true +esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2/report.json new file mode 100644 index 000000000..4a614a98f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2/report.json @@ -0,0 +1,178 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 18, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "esam_enabled": true, + "esam_rho": 0.0025, + "k": 16, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "esam_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.55613541574649, + "train_loss": 0.695133199817256, + "val_score": -11.682207347323509 + }, + { + "epoch": 1, + "test_score": -11.55939767819315, + "train_loss": 0.5532737437047457, + "val_score": -11.693448597924595 + }, + { + "epoch": 2, + "test_score": -11.555845583022986, + "train_loss": 0.5430088105954622, + "val_score": -11.695129189236418 + }, + { + "epoch": 3, + "test_score": -11.546128348237966, + "train_loss": 0.5180052236506814, + "val_score": -11.670440136243055 + }, + { + "epoch": 4, + "test_score": -11.540655833207525, + "train_loss": 0.5016618543549588, + "val_score": -11.668992019701589 + }, + { + "epoch": 5, + "test_score": -11.548262454700978, + "train_loss": 0.4916326858495411, + "val_score": -11.66722003895958 + }, + { + "epoch": 6, + "test_score": -11.580033818065433, + "train_loss": 0.47983503184820475, + "val_score": -11.701952805531674 + }, + { + "epoch": 7, + "test_score": -11.56019363082758, + "train_loss": 0.4752620725255263, + "val_score": -11.688177568511135 + }, + { + "epoch": 8, + "test_score": -11.54976600310857, + "train_loss": 0.46754956245422363, + "val_score": -11.678680192766727 + }, + { + "epoch": 9, + "test_score": -11.554276795016612, + "train_loss": 0.4644645609353718, + "val_score": -11.687640347377585 + }, + { + "epoch": 10, + "test_score": -11.528830173564812, + "train_loss": 0.45745536214426946, + "val_score": -11.665889241791978 + }, + { + "epoch": 11, + "test_score": -11.519614037300007, + "train_loss": 0.451346019380971, + "val_score": -11.658393396891343 + }, + { + "epoch": 12, + "test_score": -11.500646821780762, + "train_loss": 0.45580658002903585, + "val_score": -11.637033448665152 + }, + { + "epoch": 13, + "test_score": -11.525748576514388, + "train_loss": 0.45295710469547074, + "val_score": -11.663883276838083 + }, + { + "epoch": 14, + "test_score": -11.52637409616257, + "train_loss": 0.4448630213737488, + "val_score": -11.680014758166044 + }, + { + "epoch": 15, + "test_score": -11.498340652831377, + "train_loss": 0.443443437940196, + "val_score": -11.637611029696586 + }, + { + "epoch": 16, + "test_score": -11.527711071960198, + "train_loss": 0.43620154889006363, + "val_score": -11.678399280964193 + }, + { + "epoch": 17, + "test_score": -11.477502461081881, + "train_loss": 0.43538482251920196, + "val_score": -11.642487529267129 + }, + { + "epoch": 18, + "test_score": -11.461513970249932, + "train_loss": 0.4357229392779501, + "val_score": -11.629880475441134 + }, + { + "epoch": 19, + "test_score": -11.502000054295923, + "train_loss": 0.42684777473148544, + "val_score": -11.671822053720273 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.461513970249932, + "score": -11.461513970249932 + }, + "val": { + "rmse": 11.629880475441134, + "score": -11.629880475441134 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 16.751631021499634, + "variant": "esam_only" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0.toml new file mode 100644 index 000000000..a65899ec2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0.toml @@ -0,0 +1,29 @@ +dataset = "sberbank-housing" +variant = "mfb_only" +seed = 0 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..9f52a6eb1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0/report.json @@ -0,0 +1,118 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 3, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "mfb_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.584012530789, + "train_loss": 0.7067123651504517, + "val_score": -11.712726316433415 + }, + { + "epoch": 1, + "test_score": -11.555901041327463, + "train_loss": 0.5641985504250777, + "val_score": -11.6994959013499 + }, + { + "epoch": 2, + "test_score": -11.536238916978505, + "train_loss": 0.5335533916950226, + "val_score": -11.668051137974945 + }, + { + "epoch": 3, + "test_score": -11.527389417691648, + "train_loss": 0.5168953067377994, + "val_score": -11.649721173400192 + }, + { + "epoch": 4, + "test_score": -11.563132768791982, + "train_loss": 0.508146091511375, + "val_score": -11.67863968962139 + }, + { + "epoch": 5, + "test_score": -11.572309632036578, + "train_loss": 0.5049841780411569, + "val_score": -11.674906917748249 + }, + { + "epoch": 6, + "test_score": -11.562115307156237, + "train_loss": 0.5046039474637884, + "val_score": -11.671406972711399 + }, + { + "epoch": 7, + "test_score": -11.570539988690363, + "train_loss": 0.5026436055961409, + "val_score": -11.692601686712054 + }, + { + "epoch": 8, + "test_score": -11.58557531674288, + "train_loss": 0.48302085462369415, + "val_score": -11.692135141620707 + }, + { + "epoch": 9, + "test_score": -11.574419136385487, + "train_loss": 0.4848339212568183, + "val_score": -11.683977055104942 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.527389417691648, + "score": -11.527389417691648 + }, + "val": { + "rmse": 11.649721173400192, + "score": -11.649721173400192 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 6.568762302398682, + "variant": "mfb_only" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1.toml new file mode 100644 index 000000000..709c7b9e6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1.toml @@ -0,0 +1,29 @@ +dataset = "sberbank-housing" +variant = "mfb_only" +seed = 1 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1/report.json new file mode 100644 index 000000000..c300034d8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1/report.json @@ -0,0 +1,106 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 1, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "mfb_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.582838820918091, + "train_loss": 0.7015442032563058, + "val_score": -11.693712184891757 + }, + { + "epoch": 1, + "test_score": -11.535810359671178, + "train_loss": 0.5580546150082036, + "val_score": -11.652520531679441 + }, + { + "epoch": 2, + "test_score": -11.555896419812255, + "train_loss": 0.5397704180918241, + "val_score": -11.676539866413899 + }, + { + "epoch": 3, + "test_score": -11.589493538271325, + "train_loss": 0.5239935846705186, + "val_score": -11.7085437325865 + }, + { + "epoch": 4, + "test_score": -11.605585108597094, + "train_loss": 0.5143824502041465, + "val_score": -11.710058625919041 + }, + { + "epoch": 5, + "test_score": -11.556786357508125, + "train_loss": 0.5057109406119898, + "val_score": -11.657854803233795 + }, + { + "epoch": 6, + "test_score": -11.586644054020574, + "train_loss": 0.4948484772130063, + "val_score": -11.70785235483841 + }, + { + "epoch": 7, + "test_score": -11.586848834690507, + "train_loss": 0.497083875693773, + "val_score": -11.705622204159992 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.535810359671178, + "score": -11.535810359671178 + }, + "val": { + "rmse": 11.652520531679441, + "score": -11.652520531679441 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 5.5788445472717285, + "variant": "mfb_only" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2.toml new file mode 100644 index 000000000..db0d6a2a9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2.toml @@ -0,0 +1,29 @@ +dataset = "sberbank-housing" +variant = "mfb_only" +seed = 2 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +mfb_enabled = true +mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2/report.json new file mode 100644 index 000000000..bcd4bfc6e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2/report.json @@ -0,0 +1,124 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 4, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "mfb_enabled": true, + "mfb_keep": 0.8, + "n_bins": 48, + "n_blocks": 2 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "mfb_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.548783698174738, + "train_loss": 0.7081226235941837, + "val_score": -11.686633725624931 + }, + { + "epoch": 1, + "test_score": -11.547962514332209, + "train_loss": 0.5557862222194672, + "val_score": -11.684425643235729 + }, + { + "epoch": 2, + "test_score": -11.556265474993019, + "train_loss": 0.5497071068537863, + "val_score": -11.701433820980952 + }, + { + "epoch": 3, + "test_score": -11.557251764990456, + "train_loss": 0.5295571292701521, + "val_score": -11.683871924973317 + }, + { + "epoch": 4, + "test_score": -11.513798272472021, + "train_loss": 0.5162518667547327, + "val_score": -11.642743095889173 + }, + { + "epoch": 5, + "test_score": -11.543931722733838, + "train_loss": 0.5097392264165377, + "val_score": -11.66748356456919 + }, + { + "epoch": 6, + "test_score": -11.554824838602325, + "train_loss": 0.49350636099514206, + "val_score": -11.682919180835649 + }, + { + "epoch": 7, + "test_score": -11.545274595197622, + "train_loss": 0.49634232646540594, + "val_score": -11.684635892589156 + }, + { + "epoch": 8, + "test_score": -11.592842498385926, + "train_loss": 0.48988168176851776, + "val_score": -11.717275297835817 + }, + { + "epoch": 9, + "test_score": -11.546218213306766, + "train_loss": 0.48752505528299434, + "val_score": -11.674862480578934 + }, + { + "epoch": 10, + "test_score": -11.546426353978601, + "train_loss": 0.4796864672711021, + "val_score": -11.681482408067907 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.513798272472021, + "score": -11.513798272472021 + }, + "val": { + "rmse": 11.642743095889173, + "score": -11.642743095889173 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 8.56472897529602, + "variant": "mfb_only" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0.toml new file mode 100644 index 000000000..2856da2cf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0.toml @@ -0,0 +1,30 @@ +dataset = "sberbank-housing" +variant = "rla_only" +seed = 0 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0/report.json new file mode 100644 index 000000000..7c088e464 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0/report.json @@ -0,0 +1,113 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 2, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 0, + "variant": "rla_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.520820677453115, + "train_loss": 0.7019921698068318, + "val_score": -11.648102149766984 + }, + { + "epoch": 1, + "test_score": -11.536492208024947, + "train_loss": 0.5585583950343885, + "val_score": -11.659057606323854 + }, + { + "epoch": 2, + "test_score": -11.49868567875769, + "train_loss": 0.532463694873609, + "val_score": -11.630861178893646 + }, + { + "epoch": 3, + "test_score": -11.549928501452806, + "train_loss": 0.5084077339423331, + "val_score": -11.677087398713239 + }, + { + "epoch": 4, + "test_score": -11.576383925687757, + "train_loss": 0.4942305793887691, + "val_score": -11.696462530088892 + }, + { + "epoch": 5, + "test_score": -11.54979969197755, + "train_loss": 0.492362883530165, + "val_score": -11.66206145448411 + }, + { + "epoch": 6, + "test_score": -11.59433500145552, + "train_loss": 0.4821073114871979, + "val_score": -11.703687585568316 + }, + { + "epoch": 7, + "test_score": -11.524535832191289, + "train_loss": 0.48153042793273926, + "val_score": -11.644755323238266 + }, + { + "epoch": 8, + "test_score": -11.552281828889978, + "train_loss": 0.46902895601172195, + "val_score": -11.671850161034481 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.49868567875769, + "score": -11.49868567875769 + }, + "val": { + "rmse": 11.630861178893646, + "score": -11.630861178893646 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0", + "seed": 0, + "task_type": "regression", + "time_seconds": 6.48681378364563, + "variant": "rla_only" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1.toml new file mode 100644 index 000000000..ecff84b3e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1.toml @@ -0,0 +1,30 @@ +dataset = "sberbank-housing" +variant = "rla_only" +seed = 1 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1/report.json new file mode 100644 index 000000000..fa541d4ce --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1/report.json @@ -0,0 +1,101 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 1, + "variant": "rla_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.506723180155442, + "train_loss": 0.6959314220830014, + "val_score": -11.633652615105254 + }, + { + "epoch": 1, + "test_score": -11.540293551038493, + "train_loss": 0.5563824114046598, + "val_score": -11.683249613579397 + }, + { + "epoch": 2, + "test_score": -11.562496700801894, + "train_loss": 0.5268118506983707, + "val_score": -11.710368096147347 + }, + { + "epoch": 3, + "test_score": -11.520426645855572, + "train_loss": 0.5095094709019912, + "val_score": -11.652476009138773 + }, + { + "epoch": 4, + "test_score": -11.552124647107332, + "train_loss": 0.4992134774986066, + "val_score": -11.681942847672602 + }, + { + "epoch": 5, + "test_score": -11.545625487408978, + "train_loss": 0.49524260978949697, + "val_score": -11.674771645250711 + }, + { + "epoch": 6, + "test_score": -11.599754567673983, + "train_loss": 0.48614214439141124, + "val_score": -11.717217998818489 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.506723180155442, + "score": -11.506723180155442 + }, + "val": { + "rmse": 11.633652615105254, + "score": -11.633652615105254 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1", + "seed": 1, + "task_type": "regression", + "time_seconds": 5.13034200668335, + "variant": "rla_only" +} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2.toml new file mode 100644 index 000000000..8c2083ffb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2.toml @@ -0,0 +1,30 @@ +dataset = "sberbank-housing" +variant = "rla_only" +seed = 2 +batch_size = 1024 +patience = 6 +n_epochs = 20 +gradient_clipping_norm = 1.0 +amp = true + +[data] +path = "data/sberbank-housing" +num_policy = "standard" +cache = false + +[optimizer] +type = "AdamW" +lr = 0.0003 +weight_decay = 1e-05 + +[model] +arch_type = "tabm" +k = 16 +n_blocks = 2 +d_block = 384 +dropout = 0.1 +d_embedding = 16 +n_bins = 48 +rla_enabled = true +rla_rank = 4 +rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2/DONE new file mode 100644 index 000000000..19f86f493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2/DONE @@ -0,0 +1 @@ +done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2/report.json new file mode 100644 index 000000000..5b56c9ef9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2/report.json @@ -0,0 +1,101 @@ +{ + "amp_dtype": "bfloat16", + "amp_enabled": true, + "best_epoch": 0, + "config": { + "amp": true, + "batch_size": 1024, + "data": { + "cache": false, + "num_policy": "standard", + "path": "data/sberbank-housing" + }, + "dataset": "sberbank-housing", + "gradient_clipping_norm": 1.0, + "model": { + "arch_type": "tabm", + "d_block": 384, + "d_embedding": 16, + "dropout": 0.1, + "k": 16, + "n_bins": 48, + "n_blocks": 2, + "rla_enabled": true, + "rla_rank": 4, + "rla_scale": 0.05 + }, + "n_epochs": 20, + "optimizer": { + "lr": 0.0003, + "type": "AdamW", + "weight_decay": 1e-05 + }, + "patience": 6, + "seed": 2, + "variant": "rla_only" + }, + "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2.toml", + "dataset": "sberbank-housing", + "gpu_name": "NVIDIA GeForce RTX 4090", + "history": [ + { + "epoch": 0, + "test_score": -11.506943969685626, + "train_loss": 0.693781990753977, + "val_score": -11.638517007119077 + }, + { + "epoch": 1, + "test_score": -11.548524730973826, + "train_loss": 0.5583565109654477, + "val_score": -11.691600059995665 + }, + { + "epoch": 2, + "test_score": -11.5435219569212, + "train_loss": 0.5326070801207894, + "val_score": -11.68417294723668 + }, + { + "epoch": 3, + "test_score": -11.558364706035814, + "train_loss": 0.5135152920296318, + "val_score": -11.696133775230425 + }, + { + "epoch": 4, + "test_score": -11.55967818185129, + "train_loss": 0.49890078212085526, + "val_score": -11.692484236554279 + }, + { + "epoch": 5, + "test_score": -11.546965519985177, + "train_loss": 0.4914465530922538, + "val_score": -11.677945234810213 + }, + { + "epoch": 6, + "test_score": -11.532360819421838, + "train_loss": 0.4807174880253641, + "val_score": -11.65013244291669 + } + ], + "inference_mode": "mean", + "metric_direction": "lower", + "metrics": { + "test": { + "rmse": 11.506943969685626, + "score": -11.506943969685626 + }, + "val": { + "rmse": 11.638517007119077, + "score": -11.638517007119077 + } + }, + "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2", + "seed": 2, + "task_type": "regression", + "time_seconds": 5.9000396728515625, + "variant": "rla_only" +} diff --git a/tabm_integrated/pbs/run_integrated_16gpu.pbs b/tabm_integrated/pbs/run_integrated_16gpu.pbs new file mode 100644 index 000000000..88373c19a --- /dev/null +++ b/tabm_integrated/pbs/run_integrated_16gpu.pbs @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$PBS_O_WORKDIR" +N_GPUS=16 SEEDS="${SEEDS:-0}" ./tabm_integrated/tools/run_integrated_matrix.sh diff --git a/tabm_integrated/tools/aggregate_integrated_results.py b/tabm_integrated/tools/aggregate_integrated_results.py new file mode 100644 index 000000000..9d87935b7 --- /dev/null +++ b/tabm_integrated/tools/aggregate_integrated_results.py @@ -0,0 +1,45 @@ +from __future__ import annotations +import csv,json,math +from pathlib import Path +from statistics import mean,stdev +DATASETS=['sberbank-housing','ecom-offers','homesite-insurance','cooking-time','delivery-eta'] +VARIANTS=['baseline_plr','rla_only','esam_only','mfb_only','cf_fisd_only','all_four_combined','all_minus_rla','all_minus_esam','all_minus_mfb','all_minus_cf_fisd'] +PRIMARY={'sberbank-housing':'rmse','cooking-time':'rmse','delivery-eta':'rmse','ecom-offers':'roc-auc','homesite-insurance':'roc-auc'} +LOWER={'rmse'} +ROOT=Path(__file__).resolve().parents[1]; EXP=ROOT/'paper'/'exp'/'integrated' +def load_report(dataset,variant,seed): + path=EXP/dataset/f'{variant}-evaluation'/str(seed)/'report.json'; done=path.with_name('DONE') + if not path.exists() or not done.exists(): return None,path + r=json.loads(path.read_text()); return r,path +def delt(metric,v,b): return b-v if metric in LOWER else v-b +def fmt(x): return '' if x is None or (isinstance(x,float) and math.isnan(x)) else x +def main(): + summary=[]; audit=[] + for dataset in DATASETS: + metric=PRIMARY[dataset]; values={} + for variant in VARIANTS: + vals=[] + for seed in [0,1,2]: + r,path=load_report(dataset,variant,seed); ok=bool(r) and not r.get('failure') + audit.append({'dataset':dataset,'variant':variant,'seed':seed,'result_path':str(path.parent),'report_json_present':path.exists(),'DONE_present':(path.parent/'DONE').exists(),'failure_block_absent':ok,'gpu_name':(r or {}).get('gpu_name'),'amp_dtype':(r or {}).get('amp_dtype'),'inference_mode':(r or {}).get('inference_mode','mean'),'metric':metric,'test_metric':((r or {}).get('metrics',{}).get('test',{}).get(metric)),'failure':(r or {}).get('failure')}) + if ok: vals.append(float(r['metrics']['test'][metric])) + values[variant]=vals + base=values['baseline_plr'] + for variant in VARIANTS: + vals=values[variant] + if len(vals)==3 and len(base)==3: + m=mean(vals); s=stdev(vals); bm=mean(base); d=delt(metric,m,bm); pct=100*d/abs(bm); claim='baseline' if variant=='baseline_plr' else ('win' if d>0 else 'loss' if d<0 else 'tie') + elif vals: + m=mean(vals); s=stdev(vals) if len(vals)>1 else 0.0; bm=mean(base) if base else float('nan'); d=float('nan'); pct=float('nan'); claim='incomplete' + else: + m=s=bm=d=pct=float('nan'); claim='missing' + summary.append({'dataset':dataset,'variant':variant,'metric':metric,'metric_direction':'lower' if metric in LOWER else 'higher','inference_mode':'mean','precision':'bfloat16','n_seeds':len(vals),'mean':m,'std':s,'baseline_mean':bm,'absolute_delta':d,'percent_delta':pct,'safe_claim_status':claim,'config_path':str(EXP/dataset/f'{variant}-evaluation/0.toml'),'result_path':str(EXP/dataset/f'{variant}-evaluation')}) + for name,rows in [('final_integrated_summary.csv',summary),('final_integrated_audit.csv',audit)]: + path=ROOT/'paper'/'exp'/name; path.parent.mkdir(parents=True,exist_ok=True) + with path.open('w',newline='') as f: w=csv.DictWriter(f,fieldnames=list(rows[0].keys()),lineterminator='\n'); w.writeheader(); w.writerows(rows) + lines=['# Final Integrated TabM Experiment Report','','Mean ± std over available seeds. Safe claims require all 3 seeds.','','| dataset | variant | metric | direction | inference | precision | mean ± std | delta | percent delta | n | status | config path | result path |','|---|---|---|---|---|---|---:|---:|---:|---:|---|---|---|'] + for r in summary: + lines.append(f"| {r['dataset']} | {r['variant']} | {r['metric']} | {r['metric_direction']} | {r['inference_mode']} | {r['precision']} | {float(r['mean']):.6g} ± {float(r['std']):.3g} | {float(r['absolute_delta']):.6g} | {float(r['percent_delta']):.3f}% | {r['n_seeds']} | {r['safe_claim_status']} | {r['config_path']} | {r['result_path']} |") + (ROOT/'FINAL_EXPERIMENT_REPORT.md').write_text('\n'.join(lines)+'\n') + print(ROOT/'paper'/'exp'/'final_integrated_summary.csv'); print(ROOT/'FINAL_EXPERIMENT_REPORT.md') +if __name__=='__main__': main() diff --git a/tabm_integrated/tools/generate_integrated_configs.py b/tabm_integrated/tools/generate_integrated_configs.py new file mode 100644 index 000000000..e5ec0e0d2 --- /dev/null +++ b/tabm_integrated/tools/generate_integrated_configs.py @@ -0,0 +1,27 @@ +from __future__ import annotations +from pathlib import Path +import tomli_w +DATASETS=['sberbank-housing','ecom-offers','homesite-insurance','cooking-time','delivery-eta'] +VARIANTS={ + 'baseline_plr':{}, + 'rla_only':{'rla_enabled':True,'rla_rank':4,'rla_scale':0.05}, + 'esam_only':{'esam_enabled':True,'esam_rho':0.0025}, + 'mfb_only':{'mfb_enabled':True,'mfb_keep':0.8}, + 'cf_fisd_only':{'cf_fisd_enabled':True}, + 'all_four_combined':{'rla_enabled':True,'rla_rank':4,'rla_scale':0.05,'esam_enabled':True,'esam_rho':0.0025,'mfb_enabled':True,'mfb_keep':0.8,'cf_fisd_enabled':True}, + 'all_minus_rla':{'esam_enabled':True,'esam_rho':0.0025,'mfb_enabled':True,'mfb_keep':0.8,'cf_fisd_enabled':True}, + 'all_minus_esam':{'rla_enabled':True,'rla_rank':4,'rla_scale':0.05,'mfb_enabled':True,'mfb_keep':0.8,'cf_fisd_enabled':True}, + 'all_minus_mfb':{'rla_enabled':True,'rla_rank':4,'rla_scale':0.05,'esam_enabled':True,'esam_rho':0.0025,'cf_fisd_enabled':True}, + 'all_minus_cf_fisd':{'rla_enabled':True,'rla_rank':4,'rla_scale':0.05,'esam_enabled':True,'esam_rho':0.0025,'mfb_enabled':True,'mfb_keep':0.8}, +} +BATCH={'sberbank-housing':1024,'ecom-offers':2048,'homesite-insurance':2048,'cooking-time':2048,'delivery-eta':2048} +def cfg(dataset,variant,seed): + return {'dataset':dataset,'variant':variant,'seed':seed,'batch_size':BATCH[dataset],'patience':6,'n_epochs':20,'gradient_clipping_norm':1.0,'amp':True,'data':{'path':f'data/{dataset}','num_policy':'standard','cache':False},'optimizer':{'type':'AdamW','lr':3e-4,'weight_decay':1e-5},'model':{'arch_type':'tabm','k':16,'n_blocks':2,'d_block':384,'dropout':0.1,'d_embedding':16,'n_bins':48,**VARIANTS[variant]}} +def main(): + root=Path(__file__).resolve().parents[1]/'paper'/'exp'/'integrated' + for seed in [0,1,2]: + for dataset in DATASETS: + for variant in VARIANTS: + d=root/dataset/f'{variant}-evaluation'; d.mkdir(parents=True,exist_ok=True); (d/f'{seed}.toml').write_text(tomli_w.dumps(cfg(dataset,variant,seed))) + print(root) +if __name__=='__main__': main() diff --git a/tabm_integrated/tools/run_integrated_matrix.sh b/tabm_integrated/tools/run_integrated_matrix.sh new file mode 100755 index 000000000..8ef95f4f1 --- /dev/null +++ b/tabm_integrated/tools/run_integrated_matrix.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PAPER="$ROOT/paper" +PYTHON="${PYTHON:-/workspace/.venvs/tabm_integrated/bin/python}" +N_GPUS="${N_GPUS:-16}" +SEEDS="${SEEDS:-0}" +VARIANTS="${VARIANTS:-baseline_plr rla_only esam_only mfb_only cf_fisd_only all_four_combined}" +DATASETS="sberbank-housing ecom-offers homesite-insurance cooking-time delivery-eta" +LOG_ROOT="$PAPER/exp/integrated/_logs" +QUEUE="$PAPER/exp/integrated/_queue_${SEEDS// /_}.txt" +mkdir -p "$LOG_ROOT" +"$PYTHON" "$ROOT/tools/generate_integrated_configs.py" +: > "$QUEUE" +for seed in $SEEDS; do + for dataset in $DATASETS; do + for variant in $VARIANTS; do + cfg="$PAPER/exp/integrated/$dataset/$variant-evaluation/$seed.toml" + out="$PAPER/exp/integrated/$dataset/$variant-evaluation/$seed" + if [[ -f "$out/DONE" && -f "$out/report.json" ]] && "$PYTHON" - "$out/report.json" <<'PYCHK' +import json, sys +r=json.load(open(sys.argv[1])) +sys.exit(1 if r.get('failure') else 0) +PYCHK + then + echo "reuse $dataset $variant $seed" + else + echo "$cfg|$out|$dataset|$variant|$seed" >> "$QUEUE" + fi + done + done +done +worker() { + gpu="$1" + while true; do + line="" + exec 9<>"$QUEUE.lock" + flock 9 + if [[ -s "$QUEUE" ]]; then + line="$(head -n 1 "$QUEUE")" + tail -n +2 "$QUEUE" > "$QUEUE.tmp" + mv "$QUEUE.tmp" "$QUEUE" + fi + flock -u 9 + [[ -z "$line" ]] && break + IFS='|' read -r cfg out dataset variant seed <<< "$line" + mkdir -p "$(dirname "$out")" "$LOG_ROOT/$dataset/$variant" + log="$LOG_ROOT/$dataset/$variant/seed${seed}.gpu${gpu}.log" + echo "START $(date -Iseconds) gpu=$gpu dataset=$dataset variant=$variant seed=$seed" | tee "$log" + if ! (cd "$PAPER" && CUDA_VISIBLE_DEVICES="$gpu" "$PYTHON" "$PAPER/bin/run_integrated.py" "$cfg" "$out" --force) >> "$log" 2>&1; then + echo "FAIL dataset=$dataset variant=$variant seed=$seed log=$log" | tee -a "$log" + touch "$PAPER/exp/integrated/FAILED" + exit 1 + fi + if [[ ! -f "$out/DONE" || ! -f "$out/report.json" ]]; then + echo "FAIL missing DONE/report dataset=$dataset variant=$variant seed=$seed log=$log" | tee -a "$log" + touch "$PAPER/exp/integrated/FAILED" + exit 1 + fi + if ! "$PYTHON" - "$out/report.json" <<'PYCHK' +import json, sys +r=json.load(open(sys.argv[1])) +sys.exit(1 if r.get('failure') else 0) +PYCHK + then + echo "FAIL failure block dataset=$dataset variant=$variant seed=$seed log=$log" | tee -a "$log" + touch "$PAPER/exp/integrated/FAILED" + exit 1 + fi + echo "DONE $(date -Iseconds) gpu=$gpu dataset=$dataset variant=$variant seed=$seed" | tee -a "$log" + done +} +rm -f "$PAPER/exp/integrated/FAILED" +pids=() +for gpu in $(seq 0 $((N_GPUS - 1))); do worker "$gpu" & pids+=("$!"); done +status=0 +for pid in "${pids[@]}"; do wait "$pid" || status=1; done +"$PYTHON" "$ROOT/tools/aggregate_integrated_results.py" || status=1 +exit "$status" From 4986dfd9ecd63e08edaa0840abbf6bf9193129de Mon Sep 17 00:00:00 2001 From: abhipray-chavan Date: Sun, 26 Apr 2026 17:38:12 +0000 Subject: [PATCH 2/7] Fix integrated TabM experiments --- TEAM_MODULE_INVENTORY.md | 58 +- tabm_integrated/FINAL_EXPERIMENT_REPORT.md | 133 +- tabm_integrated/paper/.gitignore | 5 + tabm_integrated/paper/bin/run_integrated.py | 1868 +++++++++++++++-- tabm_integrated/paper/exp/.gitignore | 6 + .../paper/exp/final_integrated_audit.csv | 992 +++++++-- .../paper/exp/final_integrated_summary.csv | 82 +- .../baseline_plr-best-head-evaluation/0/DONE | 0 .../0/report.json | 147 ++ .../baseline_plr-evaluation/0.toml | 33 + .../baseline_plr-evaluation/0/DONE | 0 .../baseline_plr-evaluation/0/report.json | 143 ++ .../0/DONE | 0 .../0/report.json | 157 ++ .../baseline_plr-best-head-evaluation/0/DONE | 0 .../0/report.json | 168 ++ .../baseline_plr-evaluation/0.toml | 33 + .../baseline_plr-evaluation/0/DONE | 0 .../baseline_plr-evaluation/0/report.json | 164 ++ .../0/DONE | 0 .../0/report.json | 181 ++ .../baseline_plr-best-head-evaluation/0/DONE | 0 .../0/report.json | 219 ++ .../baseline_plr-evaluation/0.toml | 33 + .../baseline_plr-evaluation/0/DONE | 0 .../baseline_plr-evaluation/0/report.json | 215 ++ .../0/DONE | 0 .../0/report.json | 227 ++ .../baseline_plr-best-head-evaluation/0/DONE | 0 .../0/report.json | 262 +++ .../baseline_plr-evaluation/0.toml | 34 + .../baseline_plr-evaluation/0/DONE | 0 .../baseline_plr-evaluation/0/report.json | 258 +++ .../0/DONE | 0 .../0/report.json | 275 +++ .../baseline_plr-best-head-evaluation/0/DONE | 0 .../0/report.json | 190 ++ .../baseline_plr-evaluation/0.toml | 34 + .../baseline_plr-evaluation/0/DONE | 0 .../baseline_plr-evaluation/0/report.json | 186 ++ .../0/DONE | 0 .../0/report.json | 199 ++ .../all_four_combined-evaluation/0.toml | 35 - .../all_four_combined-evaluation/0/DONE | 1 - .../0/report.json | 136 -- .../all_four_combined-evaluation/1.toml | 35 - .../all_four_combined-evaluation/1/DONE | 1 - .../1/report.json | 106 - .../all_four_combined-evaluation/2.toml | 35 - .../all_four_combined-evaluation/2/DONE | 1 - .../2/report.json | 118 -- .../all_minus_cf_fisd-evaluation/0.toml | 34 - .../all_minus_cf_fisd-evaluation/0/DONE | 1 - .../0/report.json | 117 -- .../all_minus_cf_fisd-evaluation/1.toml | 34 - .../all_minus_cf_fisd-evaluation/1/DONE | 1 - .../1/report.json | 105 - .../all_minus_cf_fisd-evaluation/2.toml | 34 - .../all_minus_cf_fisd-evaluation/2/DONE | 1 - .../2/report.json | 111 - .../all_minus_esam-evaluation/0.toml | 33 - .../all_minus_esam-evaluation/0/DONE | 1 - .../all_minus_esam-evaluation/0/report.json | 182 -- .../all_minus_esam-evaluation/1.toml | 33 - .../all_minus_esam-evaluation/1/DONE | 1 - .../all_minus_esam-evaluation/1/report.json | 134 -- .../all_minus_esam-evaluation/2.toml | 33 - .../all_minus_esam-evaluation/2/DONE | 1 - .../all_minus_esam-evaluation/2/report.json | 146 -- .../all_minus_mfb-evaluation/0.toml | 33 - .../all_minus_mfb-evaluation/0/DONE | 1 - .../all_minus_mfb-evaluation/0/report.json | 122 -- .../all_minus_mfb-evaluation/1.toml | 33 - .../all_minus_mfb-evaluation/1/DONE | 1 - .../all_minus_mfb-evaluation/1/report.json | 122 -- .../all_minus_mfb-evaluation/2.toml | 33 - .../all_minus_mfb-evaluation/2/DONE | 1 - .../all_minus_mfb-evaluation/2/report.json | 110 - .../all_minus_rla-evaluation/0.toml | 32 - .../all_minus_rla-evaluation/0/DONE | 1 - .../all_minus_rla-evaluation/0/report.json | 139 -- .../all_minus_rla-evaluation/1.toml | 32 - .../all_minus_rla-evaluation/1/DONE | 1 - .../all_minus_rla-evaluation/1/report.json | 103 - .../all_minus_rla-evaluation/2.toml | 32 - .../all_minus_rla-evaluation/2/DONE | 1 - .../all_minus_rla-evaluation/2/report.json | 157 -- .../baseline_plr-evaluation/0.toml | 27 - .../baseline_plr-evaluation/0/DONE | 1 - .../baseline_plr-evaluation/0/report.json | 116 - .../baseline_plr-evaluation/1.toml | 27 - .../baseline_plr-evaluation/1/DONE | 1 - .../baseline_plr-evaluation/1/report.json | 116 - .../baseline_plr-evaluation/2.toml | 27 - .../baseline_plr-evaluation/2/DONE | 1 - .../baseline_plr-evaluation/2/report.json | 158 -- .../cf_fisd_only-evaluation/0.toml | 28 - .../cf_fisd_only-evaluation/0/DONE | 1 - .../cf_fisd_only-evaluation/0/report.json | 99 - .../cf_fisd_only-evaluation/1.toml | 28 - .../cf_fisd_only-evaluation/1/DONE | 1 - .../cf_fisd_only-evaluation/1/report.json | 117 -- .../cf_fisd_only-evaluation/2.toml | 28 - .../cf_fisd_only-evaluation/2/DONE | 1 - .../cf_fisd_only-evaluation/2/report.json | 159 -- .../cooking-time/esam_only-evaluation/0.toml | 29 - .../cooking-time/esam_only-evaluation/0/DONE | 1 - .../esam_only-evaluation/0/report.json | 118 -- .../cooking-time/esam_only-evaluation/1.toml | 29 - .../cooking-time/esam_only-evaluation/1/DONE | 1 - .../esam_only-evaluation/1/report.json | 118 -- .../cooking-time/esam_only-evaluation/2.toml | 29 - .../cooking-time/esam_only-evaluation/2/DONE | 1 - .../esam_only-evaluation/2/report.json | 160 -- .../cooking-time/mfb_only-evaluation/0.toml | 29 - .../cooking-time/mfb_only-evaluation/0/DONE | 1 - .../mfb_only-evaluation/0/report.json | 130 -- .../cooking-time/mfb_only-evaluation/1.toml | 29 - .../cooking-time/mfb_only-evaluation/1/DONE | 1 - .../mfb_only-evaluation/1/report.json | 142 -- .../cooking-time/mfb_only-evaluation/2.toml | 29 - .../cooking-time/mfb_only-evaluation/2/DONE | 1 - .../mfb_only-evaluation/2/report.json | 142 -- .../cooking-time/rla_only-evaluation/0.toml | 30 - .../cooking-time/rla_only-evaluation/0/DONE | 1 - .../rla_only-evaluation/0/report.json | 119 -- .../cooking-time/rla_only-evaluation/1.toml | 30 - .../cooking-time/rla_only-evaluation/1/DONE | 1 - .../rla_only-evaluation/1/report.json | 119 -- .../cooking-time/rla_only-evaluation/2.toml | 30 - .../cooking-time/rla_only-evaluation/2/DONE | 1 - .../rla_only-evaluation/2/report.json | 107 - .../all_four_combined-evaluation/0.toml | 35 - .../all_four_combined-evaluation/0/DONE | 1 - .../0/report.json | 106 - .../all_four_combined-evaluation/1.toml | 35 - .../all_four_combined-evaluation/1/DONE | 1 - .../1/report.json | 112 - .../all_four_combined-evaluation/2.toml | 35 - .../all_four_combined-evaluation/2/DONE | 1 - .../2/report.json | 106 - .../all_minus_cf_fisd-evaluation/0.toml | 34 - .../all_minus_cf_fisd-evaluation/0/DONE | 1 - .../0/report.json | 105 - .../all_minus_cf_fisd-evaluation/1.toml | 34 - .../all_minus_cf_fisd-evaluation/1/DONE | 1 - .../1/report.json | 111 - .../all_minus_cf_fisd-evaluation/2.toml | 34 - .../all_minus_cf_fisd-evaluation/2/DONE | 1 - .../2/report.json | 111 - .../all_minus_esam-evaluation/0.toml | 33 - .../all_minus_esam-evaluation/0/DONE | 1 - .../all_minus_esam-evaluation/0/report.json | 104 - .../all_minus_esam-evaluation/1.toml | 33 - .../all_minus_esam-evaluation/1/DONE | 1 - .../all_minus_esam-evaluation/1/report.json | 122 -- .../all_minus_esam-evaluation/2.toml | 33 - .../all_minus_esam-evaluation/2/DONE | 1 - .../all_minus_esam-evaluation/2/report.json | 104 - .../all_minus_mfb-evaluation/0.toml | 33 - .../all_minus_mfb-evaluation/0/DONE | 1 - .../all_minus_mfb-evaluation/0/report.json | 128 -- .../all_minus_mfb-evaluation/1.toml | 33 - .../all_minus_mfb-evaluation/1/DONE | 1 - .../all_minus_mfb-evaluation/1/report.json | 116 - .../all_minus_mfb-evaluation/2.toml | 33 - .../all_minus_mfb-evaluation/2/DONE | 1 - .../all_minus_mfb-evaluation/2/report.json | 104 - .../all_minus_rla-evaluation/0.toml | 32 - .../all_minus_rla-evaluation/0/DONE | 1 - .../all_minus_rla-evaluation/0/report.json | 103 - .../all_minus_rla-evaluation/1.toml | 32 - .../all_minus_rla-evaluation/1/DONE | 1 - .../all_minus_rla-evaluation/1/report.json | 109 - .../all_minus_rla-evaluation/2.toml | 32 - .../all_minus_rla-evaluation/2/DONE | 1 - .../all_minus_rla-evaluation/2/report.json | 103 - .../baseline_plr-evaluation/0.toml | 27 - .../baseline_plr-evaluation/0/DONE | 1 - .../baseline_plr-evaluation/0/report.json | 98 - .../baseline_plr-evaluation/1.toml | 27 - .../baseline_plr-evaluation/1/DONE | 1 - .../baseline_plr-evaluation/1/report.json | 98 - .../baseline_plr-evaluation/2.toml | 27 - .../baseline_plr-evaluation/2/DONE | 1 - .../baseline_plr-evaluation/2/report.json | 122 -- .../cf_fisd_only-evaluation/0.toml | 28 - .../cf_fisd_only-evaluation/0/DONE | 1 - .../cf_fisd_only-evaluation/0/report.json | 99 - .../cf_fisd_only-evaluation/1.toml | 28 - .../cf_fisd_only-evaluation/1/DONE | 1 - .../cf_fisd_only-evaluation/1/report.json | 105 - .../cf_fisd_only-evaluation/2.toml | 28 - .../cf_fisd_only-evaluation/2/DONE | 1 - .../cf_fisd_only-evaluation/2/report.json | 123 -- .../delivery-eta/esam_only-evaluation/0.toml | 29 - .../delivery-eta/esam_only-evaluation/0/DONE | 1 - .../esam_only-evaluation/0/report.json | 100 - .../delivery-eta/esam_only-evaluation/1.toml | 29 - .../delivery-eta/esam_only-evaluation/1/DONE | 1 - .../esam_only-evaluation/1/report.json | 100 - .../delivery-eta/esam_only-evaluation/2.toml | 29 - .../delivery-eta/esam_only-evaluation/2/DONE | 1 - .../esam_only-evaluation/2/report.json | 124 -- .../delivery-eta/mfb_only-evaluation/0.toml | 29 - .../delivery-eta/mfb_only-evaluation/0/DONE | 1 - .../mfb_only-evaluation/0/report.json | 100 - .../delivery-eta/mfb_only-evaluation/1.toml | 29 - .../delivery-eta/mfb_only-evaluation/1/DONE | 1 - .../mfb_only-evaluation/1/report.json | 124 -- .../delivery-eta/mfb_only-evaluation/2.toml | 29 - .../delivery-eta/mfb_only-evaluation/2/DONE | 1 - .../mfb_only-evaluation/2/report.json | 136 -- .../delivery-eta/rla_only-evaluation/0.toml | 30 - .../delivery-eta/rla_only-evaluation/0/DONE | 1 - .../rla_only-evaluation/0/report.json | 161 -- .../delivery-eta/rla_only-evaluation/1.toml | 30 - .../delivery-eta/rla_only-evaluation/1/DONE | 1 - .../rla_only-evaluation/1/report.json | 113 - .../delivery-eta/rla_only-evaluation/2.toml | 30 - .../delivery-eta/rla_only-evaluation/2/DONE | 1 - .../rla_only-evaluation/2/report.json | 101 - .../all_four_combined-evaluation/0.toml | 35 - .../all_four_combined-evaluation/0/DONE | 1 - .../0/report.json | 184 -- .../all_four_combined-evaluation/1.toml | 35 - .../all_four_combined-evaluation/1/DONE | 1 - .../1/report.json | 184 -- .../all_four_combined-evaluation/2.toml | 35 - .../all_four_combined-evaluation/2/DONE | 1 - .../2/report.json | 112 - .../all_minus_cf_fisd-evaluation/0.toml | 34 - .../all_minus_cf_fisd-evaluation/0/DONE | 1 - .../0/report.json | 105 - .../all_minus_cf_fisd-evaluation/1.toml | 34 - .../all_minus_cf_fisd-evaluation/1/DONE | 1 - .../1/report.json | 105 - .../all_minus_cf_fisd-evaluation/2.toml | 34 - .../all_minus_cf_fisd-evaluation/2/DONE | 1 - .../2/report.json | 183 -- .../all_minus_esam-evaluation/0.toml | 33 - .../all_minus_esam-evaluation/0/DONE | 1 - .../all_minus_esam-evaluation/0/report.json | 140 -- .../all_minus_esam-evaluation/1.toml | 33 - .../all_minus_esam-evaluation/1/DONE | 1 - .../all_minus_esam-evaluation/1/report.json | 110 - .../all_minus_esam-evaluation/2.toml | 33 - .../all_minus_esam-evaluation/2/DONE | 1 - .../all_minus_esam-evaluation/2/report.json | 110 - .../all_minus_mfb-evaluation/0.toml | 33 - .../all_minus_mfb-evaluation/0/DONE | 1 - .../all_minus_mfb-evaluation/0/report.json | 140 -- .../all_minus_mfb-evaluation/1.toml | 33 - .../all_minus_mfb-evaluation/1/DONE | 1 - .../all_minus_mfb-evaluation/1/report.json | 146 -- .../all_minus_mfb-evaluation/2.toml | 33 - .../all_minus_mfb-evaluation/2/DONE | 1 - .../all_minus_mfb-evaluation/2/report.json | 122 -- .../all_minus_rla-evaluation/0.toml | 32 - .../all_minus_rla-evaluation/0/DONE | 1 - .../all_minus_rla-evaluation/0/report.json | 103 - .../all_minus_rla-evaluation/1.toml | 32 - .../all_minus_rla-evaluation/1/DONE | 1 - .../all_minus_rla-evaluation/1/report.json | 133 -- .../all_minus_rla-evaluation/2.toml | 32 - .../all_minus_rla-evaluation/2/DONE | 1 - .../all_minus_rla-evaluation/2/report.json | 109 - .../baseline_plr-evaluation/0.toml | 27 - .../baseline_plr-evaluation/0/DONE | 1 - .../baseline_plr-evaluation/0/report.json | 176 -- .../baseline_plr-evaluation/1.toml | 27 - .../baseline_plr-evaluation/1/DONE | 1 - .../baseline_plr-evaluation/1/report.json | 176 -- .../baseline_plr-evaluation/2.toml | 27 - .../baseline_plr-evaluation/2/DONE | 1 - .../baseline_plr-evaluation/2/report.json | 176 -- .../cf_fisd_only-evaluation/0.toml | 28 - .../cf_fisd_only-evaluation/0/DONE | 1 - .../cf_fisd_only-evaluation/0/report.json | 141 -- .../cf_fisd_only-evaluation/1.toml | 28 - .../cf_fisd_only-evaluation/1/DONE | 1 - .../cf_fisd_only-evaluation/1/report.json | 135 -- .../cf_fisd_only-evaluation/2.toml | 28 - .../cf_fisd_only-evaluation/2/DONE | 1 - .../cf_fisd_only-evaluation/2/report.json | 135 -- .../ecom-offers/esam_only-evaluation/0.toml | 29 - .../ecom-offers/esam_only-evaluation/0/DONE | 1 - .../esam_only-evaluation/0/report.json | 178 -- .../ecom-offers/esam_only-evaluation/1.toml | 29 - .../ecom-offers/esam_only-evaluation/1/DONE | 1 - .../esam_only-evaluation/1/report.json | 178 -- .../ecom-offers/esam_only-evaluation/2.toml | 29 - .../ecom-offers/esam_only-evaluation/2/DONE | 1 - .../esam_only-evaluation/2/report.json | 178 -- .../ecom-offers/mfb_only-evaluation/0.toml | 29 - .../ecom-offers/mfb_only-evaluation/0/DONE | 1 - .../mfb_only-evaluation/0/report.json | 100 - .../ecom-offers/mfb_only-evaluation/1.toml | 29 - .../ecom-offers/mfb_only-evaluation/1/DONE | 1 - .../mfb_only-evaluation/1/report.json | 100 - .../ecom-offers/mfb_only-evaluation/2.toml | 29 - .../ecom-offers/mfb_only-evaluation/2/DONE | 1 - .../mfb_only-evaluation/2/report.json | 178 -- .../ecom-offers/rla_only-evaluation/0.toml | 30 - .../ecom-offers/rla_only-evaluation/0/DONE | 1 - .../rla_only-evaluation/0/report.json | 179 -- .../ecom-offers/rla_only-evaluation/1.toml | 30 - .../ecom-offers/rla_only-evaluation/1/DONE | 1 - .../rla_only-evaluation/1/report.json | 179 -- .../ecom-offers/rla_only-evaluation/2.toml | 30 - .../ecom-offers/rla_only-evaluation/2/DONE | 1 - .../rla_only-evaluation/2/report.json | 179 -- .../baseline_plr-best-head-evaluation/0/DONE | 0 .../0/report.json | 147 ++ .../baseline_plr-best-head-evaluation/1/DONE | 0 .../1/report.json | 147 ++ .../baseline_plr-best-head-evaluation/2/DONE | 0 .../2/report.json | 147 ++ .../baseline_plr-evaluation/0.toml | 33 + .../baseline_plr-evaluation/0/DONE | 0 .../baseline_plr-evaluation/0/report.json | 143 ++ .../baseline_plr-evaluation/1.toml | 33 + .../baseline_plr-evaluation/1/DONE | 0 .../baseline_plr-evaluation/1/report.json | 143 ++ .../baseline_plr-evaluation/2.toml | 33 + .../baseline_plr-evaluation/2/DONE | 0 .../baseline_plr-evaluation/2/report.json | 143 ++ .../0/DONE | 0 .../0/report.json | 157 ++ .../1/DONE | 0 .../1/report.json | 162 ++ .../2/DONE | 0 .../2/report.json | 157 ++ .../0/DONE | 0 .../0/report.json | 400 ++++ .../1/DONE | 0 .../1/report.json | 400 ++++ .../2/DONE | 0 .../2/report.json | 400 ++++ .../best_cf_fisd_only-evaluation/0.toml | 44 + .../best_cf_fisd_only-evaluation/0/DONE | 0 .../0/report.json | 396 ++++ .../best_cf_fisd_only-evaluation/1.toml | 44 + .../best_cf_fisd_only-evaluation/1/DONE | 0 .../1/report.json | 396 ++++ .../best_cf_fisd_only-evaluation/2.toml | 44 + .../best_cf_fisd_only-evaluation/2/DONE | 0 .../2/report.json | 396 ++++ .../0/DONE | 0 .../0/report.json | 413 ++++ .../1/DONE | 0 .../1/report.json | 412 ++++ .../2/DONE | 0 .../2/report.json | 410 ++++ .../best_combined-best-head-evaluation/0/DONE | 0 .../0/report.json | 166 ++ .../best_combined-best-head-evaluation/1/DONE | 0 .../1/report.json | 166 ++ .../best_combined-best-head-evaluation/2/DONE | 0 .../2/report.json | 166 ++ .../best_combined-evaluation/0.toml | 50 + .../best_combined-evaluation/0/DONE | 0 .../best_combined-evaluation/0/report.json | 162 ++ .../best_combined-evaluation/1.toml | 50 + .../best_combined-evaluation/1/DONE | 0 .../best_combined-evaluation/1/report.json | 162 ++ .../best_combined-evaluation/2.toml | 50 + .../best_combined-evaluation/2/DONE | 0 .../best_combined-evaluation/2/report.json | 162 ++ .../0/DONE | 0 .../0/report.json | 178 ++ .../1/DONE | 0 .../1/report.json | 180 ++ .../2/DONE | 0 .../2/report.json | 178 ++ .../0/DONE | 0 .../0/report.json | 157 ++ .../1/DONE | 0 .../1/report.json | 157 ++ .../2/DONE | 0 .../2/report.json | 157 ++ .../best_esam_only-evaluation/0.toml | 43 + .../best_esam_only-evaluation/0/DONE | 0 .../best_esam_only-evaluation/0/report.json | 153 ++ .../best_esam_only-evaluation/1.toml | 43 + .../best_esam_only-evaluation/1/DONE | 0 .../best_esam_only-evaluation/1/report.json | 153 ++ .../best_esam_only-evaluation/2.toml | 43 + .../best_esam_only-evaluation/2/DONE | 0 .../best_esam_only-evaluation/2/report.json | 153 ++ .../0/DONE | 0 .../0/report.json | 179 ++ .../1/DONE | 0 .../1/report.json | 166 ++ .../2/DONE | 0 .../2/report.json | 174 ++ .../best_mfb_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 183 ++ .../best_mfb_only-best-head-evaluation/1/DONE | 0 .../1/report.json | 183 ++ .../best_mfb_only-best-head-evaluation/2/DONE | 0 .../2/report.json | 183 ++ .../best_mfb_only-evaluation/0.toml | 46 + .../best_mfb_only-evaluation/0/DONE | 0 .../best_mfb_only-evaluation/0/report.json | 179 ++ .../best_mfb_only-evaluation/1.toml | 46 + .../best_mfb_only-evaluation/1/DONE | 0 .../best_mfb_only-evaluation/1/report.json | 179 ++ .../best_mfb_only-evaluation/2.toml | 46 + .../best_mfb_only-evaluation/2/DONE | 0 .../best_mfb_only-evaluation/2/report.json | 179 ++ .../0/DONE | 0 .../0/report.json | 193 ++ .../1/DONE | 0 .../1/report.json | 202 ++ .../2/DONE | 0 .../2/report.json | 193 ++ .../best_rla_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 156 ++ .../best_rla_only-best-head-evaluation/1/DONE | 0 .../1/report.json | 156 ++ .../best_rla_only-best-head-evaluation/2/DONE | 0 .../2/report.json | 156 ++ .../best_rla_only-evaluation/0.toml | 40 + .../best_rla_only-evaluation/0/DONE | 0 .../best_rla_only-evaluation/0/report.json | 152 ++ .../best_rla_only-evaluation/1.toml | 40 + .../best_rla_only-evaluation/1/DONE | 0 .../best_rla_only-evaluation/1/report.json | 152 ++ .../best_rla_only-evaluation/2.toml | 40 + .../best_rla_only-evaluation/2/DONE | 0 .../best_rla_only-evaluation/2/report.json | 152 ++ .../0/DONE | 0 .../0/report.json | 178 ++ .../1/DONE | 0 .../1/report.json | 169 ++ .../2/DONE | 0 .../2/report.json | 179 ++ .../baseline_plr-best-head-evaluation/0/DONE | 0 .../0/report.json | 168 ++ .../baseline_plr-best-head-evaluation/1/DONE | 0 .../1/report.json | 168 ++ .../baseline_plr-best-head-evaluation/2/DONE | 0 .../2/report.json | 168 ++ .../baseline_plr-evaluation/0.toml | 33 + .../baseline_plr-evaluation/0/DONE | 0 .../baseline_plr-evaluation/0/report.json | 164 ++ .../baseline_plr-evaluation/1.toml | 33 + .../baseline_plr-evaluation/1/DONE | 0 .../baseline_plr-evaluation/1/report.json | 164 ++ .../baseline_plr-evaluation/2.toml | 33 + .../baseline_plr-evaluation/2/DONE | 0 .../baseline_plr-evaluation/2/report.json | 164 ++ .../0/DONE | 0 .../0/report.json | 181 ++ .../1/DONE | 0 .../1/report.json | 176 ++ .../2/DONE | 0 .../2/report.json | 176 ++ .../0/DONE | 0 .../0/report.json | 449 ++++ .../1/DONE | 0 .../1/report.json | 449 ++++ .../2/DONE | 0 .../2/report.json | 449 ++++ .../best_cf_fisd_only-evaluation/0.toml | 44 + .../best_cf_fisd_only-evaluation/0/DONE | 0 .../0/report.json | 445 ++++ .../best_cf_fisd_only-evaluation/1.toml | 44 + .../best_cf_fisd_only-evaluation/1/DONE | 0 .../1/report.json | 445 ++++ .../best_cf_fisd_only-evaluation/2.toml | 44 + .../best_cf_fisd_only-evaluation/2/DONE | 0 .../2/report.json | 445 ++++ .../0/DONE | 0 .../0/report.json | 457 ++++ .../1/DONE | 0 .../1/report.json | 458 ++++ .../2/DONE | 0 .../2/report.json | 457 ++++ .../best_combined-best-head-evaluation/0/DONE | 0 .../0/report.json | 189 ++ .../best_combined-best-head-evaluation/1/DONE | 0 .../1/report.json | 189 ++ .../best_combined-best-head-evaluation/2/DONE | 0 .../2/report.json | 189 ++ .../best_combined-evaluation/0.toml | 50 + .../best_combined-evaluation/0/DONE | 0 .../best_combined-evaluation/0/report.json | 185 ++ .../best_combined-evaluation/1.toml | 50 + .../best_combined-evaluation/1/DONE | 0 .../best_combined-evaluation/1/report.json | 185 ++ .../best_combined-evaluation/2.toml | 50 + .../best_combined-evaluation/2/DONE | 0 .../best_combined-evaluation/2/report.json | 185 ++ .../0/DONE | 0 .../0/report.json | 196 ++ .../1/DONE | 0 .../1/report.json | 193 ++ .../2/DONE | 0 .../2/report.json | 200 ++ .../0/DONE | 0 .../0/report.json | 178 ++ .../1/DONE | 0 .../1/report.json | 178 ++ .../2/DONE | 0 .../2/report.json | 178 ++ .../best_esam_only-evaluation/0.toml | 43 + .../best_esam_only-evaluation/0/DONE | 0 .../best_esam_only-evaluation/0/report.json | 174 ++ .../best_esam_only-evaluation/1.toml | 43 + .../best_esam_only-evaluation/1/DONE | 0 .../best_esam_only-evaluation/1/report.json | 174 ++ .../best_esam_only-evaluation/2.toml | 43 + .../best_esam_only-evaluation/2/DONE | 0 .../best_esam_only-evaluation/2/report.json | 174 ++ .../0/DONE | 0 .../0/report.json | 190 ++ .../1/DONE | 0 .../1/report.json | 188 ++ .../2/DONE | 0 .../2/report.json | 187 ++ .../best_mfb_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 204 ++ .../best_mfb_only-best-head-evaluation/1/DONE | 0 .../1/report.json | 204 ++ .../best_mfb_only-best-head-evaluation/2/DONE | 0 .../2/report.json | 204 ++ .../best_mfb_only-evaluation/0.toml | 46 + .../best_mfb_only-evaluation/0/DONE | 0 .../best_mfb_only-evaluation/0/report.json | 200 ++ .../best_mfb_only-evaluation/1.toml | 46 + .../best_mfb_only-evaluation/1/DONE | 0 .../best_mfb_only-evaluation/1/report.json | 200 ++ .../best_mfb_only-evaluation/2.toml | 46 + .../best_mfb_only-evaluation/2/DONE | 0 .../best_mfb_only-evaluation/2/report.json | 200 ++ .../0/DONE | 0 .../0/report.json | 211 ++ .../1/DONE | 0 .../1/report.json | 211 ++ .../2/DONE | 0 .../2/report.json | 220 ++ .../best_rla_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 179 ++ .../best_rla_only-best-head-evaluation/1/DONE | 0 .../1/report.json | 179 ++ .../best_rla_only-best-head-evaluation/2/DONE | 0 .../2/report.json | 179 ++ .../best_rla_only-evaluation/0.toml | 40 + .../best_rla_only-evaluation/0/DONE | 0 .../best_rla_only-evaluation/0/report.json | 175 ++ .../best_rla_only-evaluation/1.toml | 40 + .../best_rla_only-evaluation/1/DONE | 0 .../best_rla_only-evaluation/1/report.json | 175 ++ .../best_rla_only-evaluation/2.toml | 40 + .../best_rla_only-evaluation/2/DONE | 0 .../best_rla_only-evaluation/2/report.json | 175 ++ .../0/DONE | 0 .../0/report.json | 194 ++ .../1/DONE | 0 .../1/report.json | 187 ++ .../2/DONE | 0 .../2/report.json | 189 ++ .../baseline_plr-best-head-evaluation/0/DONE | 0 .../0/report.json | 219 ++ .../baseline_plr-best-head-evaluation/1/DONE | 0 .../1/report.json | 219 ++ .../baseline_plr-best-head-evaluation/2/DONE | 0 .../2/report.json | 219 ++ .../baseline_plr-evaluation/0.toml | 33 + .../baseline_plr-evaluation/0/DONE | 0 .../baseline_plr-evaluation/0/report.json | 215 ++ .../baseline_plr-evaluation/1.toml | 33 + .../baseline_plr-evaluation/1/DONE | 0 .../baseline_plr-evaluation/1/report.json | 215 ++ .../baseline_plr-evaluation/2.toml | 33 + .../baseline_plr-evaluation/2/DONE | 0 .../baseline_plr-evaluation/2/report.json | 215 ++ .../0/DONE | 0 .../0/report.json | 227 ++ .../1/DONE | 0 .../1/report.json | 223 ++ .../2/DONE | 0 .../2/report.json | 220 ++ .../0/DONE | 0 .../0/report.json | 390 ++++ .../1/DONE | 0 .../1/report.json | 390 ++++ .../2/DONE | 0 .../2/report.json | 390 ++++ .../best_cf_fisd_only-evaluation/0.toml | 44 + .../best_cf_fisd_only-evaluation/0/DONE | 0 .../0/report.json | 386 ++++ .../best_cf_fisd_only-evaluation/1.toml | 44 + .../best_cf_fisd_only-evaluation/1/DONE | 0 .../1/report.json | 386 ++++ .../best_cf_fisd_only-evaluation/2.toml | 44 + .../best_cf_fisd_only-evaluation/2/DONE | 0 .../2/report.json | 386 ++++ .../0/DONE | 0 .../0/report.json | 391 ++++ .../1/DONE | 0 .../1/report.json | 392 ++++ .../2/DONE | 0 .../2/report.json | 392 ++++ .../best_combined-best-head-evaluation/0/DONE | 0 .../0/report.json | 426 ++++ .../best_combined-best-head-evaluation/1/DONE | 0 .../1/report.json | 426 ++++ .../best_combined-best-head-evaluation/2/DONE | 0 .../2/report.json | 426 ++++ .../best_combined-evaluation/0.toml | 57 + .../best_combined-evaluation/0/DONE | 0 .../best_combined-evaluation/0/report.json | 422 ++++ .../best_combined-evaluation/1.toml | 57 + .../best_combined-evaluation/1/DONE | 0 .../best_combined-evaluation/1/report.json | 422 ++++ .../best_combined-evaluation/2.toml | 57 + .../best_combined-evaluation/2/DONE | 0 .../best_combined-evaluation/2/report.json | 422 ++++ .../0/DONE | 0 .../0/report.json | 428 ++++ .../1/DONE | 0 .../1/report.json | 426 ++++ .../2/DONE | 0 .../2/report.json | 427 ++++ .../0/DONE | 0 .../0/report.json | 229 ++ .../1/DONE | 0 .../1/report.json | 229 ++ .../2/DONE | 0 .../2/report.json | 229 ++ .../best_esam_only-evaluation/0.toml | 43 + .../best_esam_only-evaluation/0/DONE | 0 .../best_esam_only-evaluation/0/report.json | 225 ++ .../best_esam_only-evaluation/1.toml | 43 + .../best_esam_only-evaluation/1/DONE | 0 .../best_esam_only-evaluation/1/report.json | 225 ++ .../best_esam_only-evaluation/2.toml | 43 + .../best_esam_only-evaluation/2/DONE | 0 .../best_esam_only-evaluation/2/report.json | 225 ++ .../0/DONE | 0 .../0/report.json | 230 ++ .../1/DONE | 0 .../1/report.json | 233 ++ .../2/DONE | 0 .../2/report.json | 230 ++ .../best_mfb_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 255 +++ .../best_mfb_only-best-head-evaluation/1/DONE | 0 .../1/report.json | 255 +++ .../best_mfb_only-best-head-evaluation/2/DONE | 0 .../2/report.json | 255 +++ .../best_mfb_only-evaluation/0.toml | 46 + .../best_mfb_only-evaluation/0/DONE | 0 .../best_mfb_only-evaluation/0/report.json | 251 +++ .../best_mfb_only-evaluation/1.toml | 46 + .../best_mfb_only-evaluation/1/DONE | 0 .../best_mfb_only-evaluation/1/report.json | 251 +++ .../best_mfb_only-evaluation/2.toml | 46 + .../best_mfb_only-evaluation/2/DONE | 0 .../best_mfb_only-evaluation/2/report.json | 251 +++ .../0/DONE | 0 .../0/report.json | 255 +++ .../1/DONE | 0 .../1/report.json | 256 +++ .../2/DONE | 0 .../2/report.json | 255 +++ .../best_rla_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 228 ++ .../best_rla_only-best-head-evaluation/1/DONE | 0 .../1/report.json | 228 ++ .../best_rla_only-best-head-evaluation/2/DONE | 0 .../2/report.json | 228 ++ .../best_rla_only-evaluation/0.toml | 40 + .../best_rla_only-evaluation/0/DONE | 0 .../best_rla_only-evaluation/0/report.json | 224 ++ .../best_rla_only-evaluation/1.toml | 40 + .../best_rla_only-evaluation/1/DONE | 0 .../best_rla_only-evaluation/1/report.json | 224 ++ .../best_rla_only-evaluation/2.toml | 40 + .../best_rla_only-evaluation/2/DONE | 0 .../best_rla_only-evaluation/2/report.json | 224 ++ .../0/DONE | 0 .../0/report.json | 228 ++ .../1/DONE | 0 .../1/report.json | 232 ++ .../2/DONE | 0 .../2/report.json | 229 ++ .../baseline_plr-best-head-evaluation/0/DONE | 0 .../0/report.json | 262 +++ .../baseline_plr-best-head-evaluation/1/DONE | 0 .../1/report.json | 262 +++ .../baseline_plr-best-head-evaluation/2/DONE | 0 .../2/report.json | 262 +++ .../baseline_plr-evaluation/0.toml | 34 + .../baseline_plr-evaluation/0/DONE | 0 .../baseline_plr-evaluation/0/report.json | 258 +++ .../baseline_plr-evaluation/1.toml | 34 + .../baseline_plr-evaluation/1/DONE | 0 .../baseline_plr-evaluation/1/report.json | 258 +++ .../baseline_plr-evaluation/2.toml | 34 + .../baseline_plr-evaluation/2/DONE | 0 .../baseline_plr-evaluation/2/report.json | 258 +++ .../0/DONE | 0 .../0/report.json | 275 +++ .../1/DONE | 0 .../1/report.json | 277 +++ .../2/DONE | 0 .../2/report.json | 269 +++ .../0/DONE | 0 .../0/report.json | 621 ++++++ .../1/DONE | 0 .../1/report.json | 621 ++++++ .../2/DONE | 0 .../2/report.json | 621 ++++++ .../best_cf_fisd_only-evaluation/0.toml | 45 + .../best_cf_fisd_only-evaluation/0/DONE | 0 .../0/report.json | 617 ++++++ .../best_cf_fisd_only-evaluation/1.toml | 45 + .../best_cf_fisd_only-evaluation/1/DONE | 0 .../1/report.json | 617 ++++++ .../best_cf_fisd_only-evaluation/2.toml | 45 + .../best_cf_fisd_only-evaluation/2/DONE | 0 .../2/report.json | 617 ++++++ .../0/DONE | 0 .../0/report.json | 639 ++++++ .../1/DONE | 0 .../1/report.json | 631 ++++++ .../2/DONE | 0 .../2/report.json | 626 ++++++ .../best_combined-best-head-evaluation/0/DONE | 0 .../0/report.json | 657 ++++++ .../best_combined-best-head-evaluation/1/DONE | 0 .../1/report.json | 657 ++++++ .../best_combined-best-head-evaluation/2/DONE | 0 .../2/report.json | 657 ++++++ .../best_combined-evaluation/0.toml | 58 + .../best_combined-evaluation/0/DONE | 0 .../best_combined-evaluation/0/report.json | 653 ++++++ .../best_combined-evaluation/1.toml | 58 + .../best_combined-evaluation/1/DONE | 0 .../best_combined-evaluation/1/report.json | 653 ++++++ .../best_combined-evaluation/2.toml | 58 + .../best_combined-evaluation/2/DONE | 0 .../best_combined-evaluation/2/report.json | 653 ++++++ .../0/DONE | 0 .../0/report.json | 666 ++++++ .../1/DONE | 0 .../1/report.json | 671 ++++++ .../2/DONE | 0 .../2/report.json | 667 ++++++ .../0/DONE | 0 .../0/report.json | 272 +++ .../1/DONE | 0 .../1/report.json | 272 +++ .../2/DONE | 0 .../2/report.json | 272 +++ .../best_esam_only-evaluation/0.toml | 44 + .../best_esam_only-evaluation/0/DONE | 0 .../best_esam_only-evaluation/0/report.json | 268 +++ .../best_esam_only-evaluation/1.toml | 44 + .../best_esam_only-evaluation/1/DONE | 0 .../best_esam_only-evaluation/1/report.json | 268 +++ .../best_esam_only-evaluation/2.toml | 44 + .../best_esam_only-evaluation/2/DONE | 0 .../best_esam_only-evaluation/2/report.json | 268 +++ .../0/DONE | 0 .../0/report.json | 290 +++ .../1/DONE | 0 .../1/report.json | 278 +++ .../2/DONE | 0 .../2/report.json | 283 +++ .../best_mfb_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 298 +++ .../best_mfb_only-best-head-evaluation/1/DONE | 0 .../1/report.json | 298 +++ .../best_mfb_only-best-head-evaluation/2/DONE | 0 .../2/report.json | 298 +++ .../best_mfb_only-evaluation/0.toml | 47 + .../best_mfb_only-evaluation/0/DONE | 0 .../best_mfb_only-evaluation/0/report.json | 294 +++ .../best_mfb_only-evaluation/1.toml | 47 + .../best_mfb_only-evaluation/1/DONE | 0 .../best_mfb_only-evaluation/1/report.json | 294 +++ .../best_mfb_only-evaluation/2.toml | 47 + .../best_mfb_only-evaluation/2/DONE | 0 .../best_mfb_only-evaluation/2/report.json | 294 +++ .../0/DONE | 0 .../0/report.json | 304 +++ .../1/DONE | 0 .../1/report.json | 310 +++ .../2/DONE | 0 .../2/report.json | 309 +++ .../best_rla_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 275 +++ .../best_rla_only-best-head-evaluation/1/DONE | 0 .../1/report.json | 275 +++ .../best_rla_only-best-head-evaluation/2/DONE | 0 .../2/report.json | 275 +++ .../best_rla_only-evaluation/0.toml | 41 + .../best_rla_only-evaluation/0/DONE | 0 .../best_rla_only-evaluation/0/report.json | 271 +++ .../best_rla_only-evaluation/1.toml | 41 + .../best_rla_only-evaluation/1/DONE | 0 .../best_rla_only-evaluation/1/report.json | 271 +++ .../best_rla_only-evaluation/2.toml | 41 + .../best_rla_only-evaluation/2/DONE | 0 .../best_rla_only-evaluation/2/report.json | 271 +++ .../0/DONE | 0 .../0/report.json | 284 +++ .../1/DONE | 0 .../1/report.json | 284 +++ .../2/DONE | 0 .../2/report.json | 293 +++ .../baseline_plr-best-head-evaluation/0/DONE | 0 .../0/report.json | 190 ++ .../baseline_plr-best-head-evaluation/1/DONE | 0 .../1/report.json | 190 ++ .../baseline_plr-best-head-evaluation/2/DONE | 0 .../2/report.json | 190 ++ .../baseline_plr-evaluation/0.toml | 34 + .../baseline_plr-evaluation/0/DONE | 0 .../baseline_plr-evaluation/0/report.json | 186 ++ .../baseline_plr-evaluation/1.toml | 34 + .../baseline_plr-evaluation/1/DONE | 0 .../baseline_plr-evaluation/1/report.json | 186 ++ .../baseline_plr-evaluation/2.toml | 34 + .../baseline_plr-evaluation/2/DONE | 0 .../baseline_plr-evaluation/2/report.json | 186 ++ .../0/DONE | 0 .../0/report.json | 199 ++ .../1/DONE | 0 .../1/report.json | 200 ++ .../2/DONE | 0 .../2/report.json | 201 ++ .../0/DONE | 0 .../0/report.json | 643 ++++++ .../1/DONE | 0 .../1/report.json | 643 ++++++ .../2/DONE | 0 .../2/report.json | 643 ++++++ .../best_cf_fisd_only-evaluation/0.toml | 45 + .../best_cf_fisd_only-evaluation/0/DONE | 0 .../0/report.json | 639 ++++++ .../best_cf_fisd_only-evaluation/1.toml | 45 + .../best_cf_fisd_only-evaluation/1/DONE | 0 .../1/report.json | 639 ++++++ .../best_cf_fisd_only-evaluation/2.toml | 45 + .../best_cf_fisd_only-evaluation/2/DONE | 0 .../2/report.json | 639 ++++++ .../0/DONE | 0 .../0/report.json | 649 ++++++ .../1/DONE | 0 .../1/report.json | 658 ++++++ .../2/DONE | 0 .../2/report.json | 648 ++++++ .../best_combined-best-head-evaluation/0/DONE | 0 .../0/report.json | 679 ++++++ .../best_combined-best-head-evaluation/1/DONE | 0 .../1/report.json | 679 ++++++ .../best_combined-best-head-evaluation/2/DONE | 0 .../2/report.json | 679 ++++++ .../best_combined-evaluation/0.toml | 58 + .../best_combined-evaluation/0/DONE | 0 .../best_combined-evaluation/0/report.json | 675 ++++++ .../best_combined-evaluation/1.toml | 58 + .../best_combined-evaluation/1/DONE | 0 .../best_combined-evaluation/1/report.json | 675 ++++++ .../best_combined-evaluation/2.toml | 58 + .../best_combined-evaluation/2/DONE | 0 .../best_combined-evaluation/2/report.json | 675 ++++++ .../0/DONE | 0 .../0/report.json | 688 ++++++ .../1/DONE | 0 .../1/report.json | 683 ++++++ .../2/DONE | 0 .../2/report.json | 687 ++++++ .../0/DONE | 0 .../0/report.json | 200 ++ .../1/DONE | 0 .../1/report.json | 200 ++ .../2/DONE | 0 .../2/report.json | 200 ++ .../best_esam_only-evaluation/0.toml | 44 + .../best_esam_only-evaluation/0/DONE | 0 .../best_esam_only-evaluation/0/report.json | 196 ++ .../best_esam_only-evaluation/1.toml | 44 + .../best_esam_only-evaluation/1/DONE | 0 .../best_esam_only-evaluation/1/report.json | 196 ++ .../best_esam_only-evaluation/2.toml | 44 + .../best_esam_only-evaluation/2/DONE | 0 .../best_esam_only-evaluation/2/report.json | 196 ++ .../0/DONE | 0 .../0/report.json | 207 ++ .../1/DONE | 0 .../1/report.json | 206 ++ .../2/DONE | 0 .../2/report.json | 205 ++ .../best_mfb_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 226 ++ .../best_mfb_only-best-head-evaluation/1/DONE | 0 .../1/report.json | 226 ++ .../best_mfb_only-best-head-evaluation/2/DONE | 0 .../2/report.json | 226 ++ .../best_mfb_only-evaluation/0.toml | 47 + .../best_mfb_only-evaluation/0/DONE | 0 .../best_mfb_only-evaluation/0/report.json | 222 ++ .../best_mfb_only-evaluation/1.toml | 47 + .../best_mfb_only-evaluation/1/DONE | 0 .../best_mfb_only-evaluation/1/report.json | 222 ++ .../best_mfb_only-evaluation/2.toml | 47 + .../best_mfb_only-evaluation/2/DONE | 0 .../best_mfb_only-evaluation/2/report.json | 222 ++ .../0/DONE | 0 .../0/report.json | 235 +++ .../1/DONE | 0 .../1/report.json | 235 +++ .../2/DONE | 0 .../2/report.json | 238 +++ .../best_rla_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 203 ++ .../best_rla_only-best-head-evaluation/1/DONE | 0 .../1/report.json | 203 ++ .../best_rla_only-best-head-evaluation/2/DONE | 0 .../2/report.json | 203 ++ .../best_rla_only-evaluation/0.toml | 41 + .../best_rla_only-evaluation/0/DONE | 0 .../best_rla_only-evaluation/0/report.json | 199 ++ .../best_rla_only-evaluation/1.toml | 41 + .../best_rla_only-evaluation/1/DONE | 0 .../best_rla_only-evaluation/1/report.json | 199 ++ .../best_rla_only-evaluation/2.toml | 41 + .../best_rla_only-evaluation/2/DONE | 0 .../best_rla_only-evaluation/2/report.json | 199 ++ .../0/DONE | 0 .../0/report.json | 206 ++ .../1/DONE | 0 .../1/report.json | 206 ++ .../2/DONE | 0 .../2/report.json | 215 ++ .../all_four_combined-evaluation/0.toml | 35 - .../all_four_combined-evaluation/0/DONE | 1 - .../0/report.json | 184 -- .../all_four_combined-evaluation/1.toml | 35 - .../all_four_combined-evaluation/1/DONE | 1 - .../1/report.json | 184 -- .../all_four_combined-evaluation/2.toml | 35 - .../all_four_combined-evaluation/2/DONE | 1 - .../2/report.json | 184 -- .../all_minus_cf_fisd-evaluation/0.toml | 34 - .../all_minus_cf_fisd-evaluation/0/DONE | 1 - .../0/report.json | 183 -- .../all_minus_cf_fisd-evaluation/1.toml | 34 - .../all_minus_cf_fisd-evaluation/1/DONE | 1 - .../1/report.json | 183 -- .../all_minus_cf_fisd-evaluation/2.toml | 34 - .../all_minus_cf_fisd-evaluation/2/DONE | 1 - .../2/report.json | 183 -- .../all_minus_esam-evaluation/0.toml | 33 - .../all_minus_esam-evaluation/0/DONE | 1 - .../all_minus_esam-evaluation/0/report.json | 182 -- .../all_minus_esam-evaluation/1.toml | 33 - .../all_minus_esam-evaluation/1/DONE | 1 - .../all_minus_esam-evaluation/1/report.json | 182 -- .../all_minus_esam-evaluation/2.toml | 33 - .../all_minus_esam-evaluation/2/DONE | 1 - .../all_minus_esam-evaluation/2/report.json | 182 -- .../all_minus_mfb-evaluation/0.toml | 33 - .../all_minus_mfb-evaluation/0/DONE | 1 - .../all_minus_mfb-evaluation/0/report.json | 182 -- .../all_minus_mfb-evaluation/1.toml | 33 - .../all_minus_mfb-evaluation/1/DONE | 1 - .../all_minus_mfb-evaluation/1/report.json | 182 -- .../all_minus_mfb-evaluation/2.toml | 33 - .../all_minus_mfb-evaluation/2/DONE | 1 - .../all_minus_mfb-evaluation/2/report.json | 182 -- .../all_minus_rla-evaluation/0.toml | 32 - .../all_minus_rla-evaluation/0/DONE | 1 - .../all_minus_rla-evaluation/0/report.json | 181 -- .../all_minus_rla-evaluation/1.toml | 32 - .../all_minus_rla-evaluation/1/DONE | 1 - .../all_minus_rla-evaluation/1/report.json | 181 -- .../all_minus_rla-evaluation/2.toml | 32 - .../all_minus_rla-evaluation/2/DONE | 1 - .../all_minus_rla-evaluation/2/report.json | 181 -- .../baseline_plr-evaluation/0.toml | 27 - .../baseline_plr-evaluation/0/DONE | 1 - .../baseline_plr-evaluation/0/report.json | 176 -- .../baseline_plr-evaluation/1.toml | 27 - .../baseline_plr-evaluation/1/DONE | 1 - .../baseline_plr-evaluation/1/report.json | 176 -- .../baseline_plr-evaluation/2.toml | 27 - .../baseline_plr-evaluation/2/DONE | 1 - .../baseline_plr-evaluation/2/report.json | 176 -- .../cf_fisd_only-evaluation/0.toml | 28 - .../cf_fisd_only-evaluation/0/DONE | 1 - .../cf_fisd_only-evaluation/0/report.json | 177 -- .../cf_fisd_only-evaluation/1.toml | 28 - .../cf_fisd_only-evaluation/1/DONE | 1 - .../cf_fisd_only-evaluation/1/report.json | 177 -- .../cf_fisd_only-evaluation/2.toml | 28 - .../cf_fisd_only-evaluation/2/DONE | 1 - .../cf_fisd_only-evaluation/2/report.json | 177 -- .../esam_only-evaluation/0.toml | 29 - .../esam_only-evaluation/0/DONE | 1 - .../esam_only-evaluation/0/report.json | 178 -- .../esam_only-evaluation/1.toml | 29 - .../esam_only-evaluation/1/DONE | 1 - .../esam_only-evaluation/1/report.json | 178 -- .../esam_only-evaluation/2.toml | 29 - .../esam_only-evaluation/2/DONE | 1 - .../esam_only-evaluation/2/report.json | 178 -- .../mfb_only-evaluation/0.toml | 29 - .../mfb_only-evaluation/0/DONE | 1 - .../mfb_only-evaluation/0/report.json | 178 -- .../mfb_only-evaluation/1.toml | 29 - .../mfb_only-evaluation/1/DONE | 1 - .../mfb_only-evaluation/1/report.json | 178 -- .../mfb_only-evaluation/2.toml | 29 - .../mfb_only-evaluation/2/DONE | 1 - .../mfb_only-evaluation/2/report.json | 178 -- .../rla_only-evaluation/0.toml | 30 - .../rla_only-evaluation/0/DONE | 1 - .../rla_only-evaluation/0/report.json | 179 -- .../rla_only-evaluation/1.toml | 30 - .../rla_only-evaluation/1/DONE | 1 - .../rla_only-evaluation/1/report.json | 179 -- .../rla_only-evaluation/2.toml | 30 - .../rla_only-evaluation/2/DONE | 1 - .../rla_only-evaluation/2/report.json | 179 -- .../integrated/manifest_baseline_fidelity.txt | 5 + .../paper/exp/integrated/manifest_final.txt | 90 + .../paper/exp/integrated/manifest_smoke.txt | 30 + .../paper/exp/integrated/manifest_sweeps.txt | 155 ++ .../integrated/manifest_sweeps_remaining.txt | 18 + .../all_four_combined-evaluation/0.toml | 35 - .../all_four_combined-evaluation/0/DONE | 1 - .../0/report.json | 106 - .../all_four_combined-evaluation/1.toml | 35 - .../all_four_combined-evaluation/1/DONE | 1 - .../1/report.json | 106 - .../all_four_combined-evaluation/2.toml | 35 - .../all_four_combined-evaluation/2/DONE | 1 - .../2/report.json | 106 - .../all_minus_cf_fisd-evaluation/0.toml | 34 - .../all_minus_cf_fisd-evaluation/0/DONE | 1 - .../0/report.json | 117 -- .../all_minus_cf_fisd-evaluation/1.toml | 34 - .../all_minus_cf_fisd-evaluation/1/DONE | 1 - .../1/report.json | 105 - .../all_minus_cf_fisd-evaluation/2.toml | 34 - .../all_minus_cf_fisd-evaluation/2/DONE | 1 - .../2/report.json | 105 - .../all_minus_esam-evaluation/0.toml | 33 - .../all_minus_esam-evaluation/0/DONE | 1 - .../all_minus_esam-evaluation/0/report.json | 104 - .../all_minus_esam-evaluation/1.toml | 33 - .../all_minus_esam-evaluation/1/DONE | 1 - .../all_minus_esam-evaluation/1/report.json | 104 - .../all_minus_esam-evaluation/2.toml | 33 - .../all_minus_esam-evaluation/2/DONE | 1 - .../all_minus_esam-evaluation/2/report.json | 104 - .../all_minus_mfb-evaluation/0.toml | 33 - .../all_minus_mfb-evaluation/0/DONE | 1 - .../all_minus_mfb-evaluation/0/report.json | 104 - .../all_minus_mfb-evaluation/1.toml | 33 - .../all_minus_mfb-evaluation/1/DONE | 1 - .../all_minus_mfb-evaluation/1/report.json | 104 - .../all_minus_mfb-evaluation/2.toml | 33 - .../all_minus_mfb-evaluation/2/DONE | 1 - .../all_minus_mfb-evaluation/2/report.json | 104 - .../all_minus_rla-evaluation/0.toml | 32 - .../all_minus_rla-evaluation/0/DONE | 1 - .../all_minus_rla-evaluation/0/report.json | 127 -- .../all_minus_rla-evaluation/1.toml | 32 - .../all_minus_rla-evaluation/1/DONE | 1 - .../all_minus_rla-evaluation/1/report.json | 133 -- .../all_minus_rla-evaluation/2.toml | 32 - .../all_minus_rla-evaluation/2/DONE | 1 - .../all_minus_rla-evaluation/2/report.json | 103 - .../baseline_plr-evaluation/0.toml | 27 - .../baseline_plr-evaluation/0/DONE | 1 - .../baseline_plr-evaluation/0/report.json | 128 -- .../baseline_plr-evaluation/1.toml | 27 - .../baseline_plr-evaluation/1/DONE | 1 - .../baseline_plr-evaluation/1/report.json | 104 - .../baseline_plr-evaluation/2.toml | 27 - .../baseline_plr-evaluation/2/DONE | 1 - .../baseline_plr-evaluation/2/report.json | 176 -- .../cf_fisd_only-evaluation/0.toml | 28 - .../cf_fisd_only-evaluation/0/DONE | 1 - .../cf_fisd_only-evaluation/0/report.json | 129 -- .../cf_fisd_only-evaluation/1.toml | 28 - .../cf_fisd_only-evaluation/1/DONE | 1 - .../cf_fisd_only-evaluation/1/report.json | 105 - .../cf_fisd_only-evaluation/2.toml | 28 - .../cf_fisd_only-evaluation/2/DONE | 1 - .../cf_fisd_only-evaluation/2/report.json | 99 - .../esam_only-evaluation/0.toml | 29 - .../esam_only-evaluation/0/DONE | 1 - .../esam_only-evaluation/0/report.json | 130 -- .../esam_only-evaluation/1.toml | 29 - .../esam_only-evaluation/1/DONE | 1 - .../esam_only-evaluation/1/report.json | 106 - .../esam_only-evaluation/2.toml | 29 - .../esam_only-evaluation/2/DONE | 1 - .../esam_only-evaluation/2/report.json | 178 -- .../mfb_only-evaluation/0.toml | 29 - .../mfb_only-evaluation/0/DONE | 1 - .../mfb_only-evaluation/0/report.json | 118 -- .../mfb_only-evaluation/1.toml | 29 - .../mfb_only-evaluation/1/DONE | 1 - .../mfb_only-evaluation/1/report.json | 106 - .../mfb_only-evaluation/2.toml | 29 - .../mfb_only-evaluation/2/DONE | 1 - .../mfb_only-evaluation/2/report.json | 124 -- .../rla_only-evaluation/0.toml | 30 - .../rla_only-evaluation/0/DONE | 1 - .../rla_only-evaluation/0/report.json | 113 - .../rla_only-evaluation/1.toml | 30 - .../rla_only-evaluation/1/DONE | 1 - .../rla_only-evaluation/1/report.json | 101 - .../rla_only-evaluation/2.toml | 30 - .../rla_only-evaluation/2/DONE | 1 - .../rla_only-evaluation/2/report.json | 101 - .../0/DONE | 0 .../0/report.json | 455 ++++ .../all_four_combined-evaluation/0.toml | 74 + .../all_four_combined-evaluation/0/DONE | 0 .../0/report.json | 451 ++++ .../0/DONE | 0 .../0/report.json | 464 ++++ .../baseline_plr-best-head-evaluation/0/DONE | 0 .../0/report.json | 147 ++ .../baseline_plr-evaluation/0.toml | 33 + .../baseline_plr-evaluation/0/DONE | 0 .../baseline_plr-evaluation/0/report.json | 143 ++ .../0/DONE | 0 .../0/report.json | 157 ++ .../cf_fisd_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 400 ++++ .../cf_fisd_only-evaluation/0.toml | 44 + .../cf_fisd_only-evaluation/0/DONE | 0 .../cf_fisd_only-evaluation/0/report.json | 396 ++++ .../0/DONE | 0 .../0/report.json | 403 ++++ .../esam_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 157 ++ .../cooking-time/esam_only-evaluation/0.toml | 43 + .../cooking-time/esam_only-evaluation/0/DONE | 0 .../esam_only-evaluation/0/report.json | 153 ++ .../esam_only-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 167 ++ .../mfb_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 183 ++ .../cooking-time/mfb_only-evaluation/0.toml | 46 + .../cooking-time/mfb_only-evaluation/0/DONE | 0 .../mfb_only-evaluation/0/report.json | 179 ++ .../mfb_only-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 193 ++ .../rla_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 156 ++ .../cooking-time/rla_only-evaluation/0.toml | 40 + .../cooking-time/rla_only-evaluation/0/DONE | 0 .../rla_only-evaluation/0/report.json | 152 ++ .../rla_only-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 177 ++ .../0/DONE | 0 .../0/report.json | 506 +++++ .../all_four_combined-evaluation/0.toml | 74 + .../all_four_combined-evaluation/0/DONE | 0 .../0/report.json | 502 +++++ .../0/DONE | 0 .../0/report.json | 510 +++++ .../baseline_plr-best-head-evaluation/0/DONE | 0 .../0/report.json | 168 ++ .../baseline_plr-evaluation/0.toml | 33 + .../baseline_plr-evaluation/0/DONE | 0 .../baseline_plr-evaluation/0/report.json | 164 ++ .../0/DONE | 0 .../0/report.json | 181 ++ .../cf_fisd_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 449 ++++ .../cf_fisd_only-evaluation/0.toml | 44 + .../cf_fisd_only-evaluation/0/DONE | 0 .../cf_fisd_only-evaluation/0/report.json | 445 ++++ .../0/DONE | 0 .../0/report.json | 469 +++++ .../esam_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 178 ++ .../delivery-eta/esam_only-evaluation/0.toml | 43 + .../delivery-eta/esam_only-evaluation/0/DONE | 0 .../esam_only-evaluation/0/report.json | 174 ++ .../esam_only-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 190 ++ .../mfb_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 204 ++ .../delivery-eta/mfb_only-evaluation/0.toml | 46 + .../delivery-eta/mfb_only-evaluation/0/DONE | 0 .../mfb_only-evaluation/0/report.json | 200 ++ .../mfb_only-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 217 ++ .../rla_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 179 ++ .../delivery-eta/rla_only-evaluation/0.toml | 40 + .../delivery-eta/rla_only-evaluation/0/DONE | 0 .../rla_only-evaluation/0/report.json | 175 ++ .../rla_only-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 184 ++ .../0/DONE | 0 .../0/report.json | 445 ++++ .../all_four_combined-evaluation/0.toml | 74 + .../all_four_combined-evaluation/0/DONE | 0 .../0/report.json | 441 ++++ .../0/DONE | 0 .../0/report.json | 447 ++++ .../baseline_plr-best-head-evaluation/0/DONE | 0 .../0/report.json | 219 ++ .../baseline_plr-evaluation/0.toml | 33 + .../baseline_plr-evaluation/0/DONE | 0 .../baseline_plr-evaluation/0/report.json | 215 ++ .../0/DONE | 0 .../0/report.json | 227 ++ .../cf_fisd_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 390 ++++ .../cf_fisd_only-evaluation/0.toml | 44 + .../cf_fisd_only-evaluation/0/DONE | 0 .../cf_fisd_only-evaluation/0/report.json | 386 ++++ .../0/DONE | 0 .../0/report.json | 391 ++++ .../esam_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 229 ++ .../ecom-offers/esam_only-evaluation/0.toml | 43 + .../ecom-offers/esam_only-evaluation/0/DONE | 0 .../esam_only-evaluation/0/report.json | 225 ++ .../esam_only-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 230 ++ .../mfb_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 255 +++ .../ecom-offers/mfb_only-evaluation/0.toml | 46 + .../ecom-offers/mfb_only-evaluation/0/DONE | 0 .../mfb_only-evaluation/0/report.json | 251 +++ .../mfb_only-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 256 +++ .../rla_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 228 ++ .../ecom-offers/rla_only-evaluation/0.toml | 40 + .../ecom-offers/rla_only-evaluation/0/DONE | 0 .../rla_only-evaluation/0/report.json | 224 ++ .../rla_only-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 228 ++ .../0/DONE | 0 .../0/report.json | 680 ++++++ .../all_four_combined-evaluation/0.toml | 75 + .../all_four_combined-evaluation/0/DONE | 0 .../0/report.json | 676 ++++++ .../0/DONE | 0 .../0/report.json | 690 ++++++ .../baseline_plr-best-head-evaluation/0/DONE | 0 .../0/report.json | 262 +++ .../baseline_plr-evaluation/0.toml | 34 + .../baseline_plr-evaluation/0/DONE | 0 .../baseline_plr-evaluation/0/report.json | 258 +++ .../0/DONE | 0 .../0/report.json | 275 +++ .../cf_fisd_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 621 ++++++ .../cf_fisd_only-evaluation/0.toml | 45 + .../cf_fisd_only-evaluation/0/DONE | 0 .../cf_fisd_only-evaluation/0/report.json | 617 ++++++ .../0/DONE | 0 .../0/report.json | 639 ++++++ .../esam_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 272 +++ .../esam_only-evaluation/0.toml | 44 + .../esam_only-evaluation/0/DONE | 0 .../esam_only-evaluation/0/report.json | 268 +++ .../esam_only-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 290 +++ .../mfb_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 298 +++ .../mfb_only-evaluation/0.toml | 47 + .../mfb_only-evaluation/0/DONE | 0 .../mfb_only-evaluation/0/report.json | 294 +++ .../mfb_only-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 307 +++ .../rla_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 275 +++ .../rla_only-evaluation/0.toml | 41 + .../rla_only-evaluation/0/DONE | 0 .../rla_only-evaluation/0/report.json | 271 +++ .../rla_only-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 285 +++ .../0/DONE | 0 .../0/report.json | 702 +++++++ .../all_four_combined-evaluation/0.toml | 75 + .../all_four_combined-evaluation/0/DONE | 0 .../0/report.json | 698 ++++++ .../0/DONE | 0 .../0/report.json | 710 +++++++ .../baseline_plr-best-head-evaluation/0/DONE | 0 .../0/report.json | 190 ++ .../baseline_plr-evaluation/0.toml | 34 + .../baseline_plr-evaluation/0/DONE | 0 .../baseline_plr-evaluation/0/report.json | 186 ++ .../0/DONE | 0 .../0/report.json | 199 ++ .../cf_fisd_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 643 ++++++ .../cf_fisd_only-evaluation/0.toml | 45 + .../cf_fisd_only-evaluation/0/DONE | 0 .../cf_fisd_only-evaluation/0/report.json | 639 ++++++ .../0/DONE | 0 .../0/report.json | 649 ++++++ .../esam_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 200 ++ .../esam_only-evaluation/0.toml | 44 + .../esam_only-evaluation/0/DONE | 0 .../esam_only-evaluation/0/report.json | 196 ++ .../esam_only-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 209 ++ .../mfb_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 226 ++ .../mfb_only-evaluation/0.toml | 47 + .../mfb_only-evaluation/0/DONE | 0 .../mfb_only-evaluation/0/report.json | 222 ++ .../mfb_only-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 234 +++ .../rla_only-best-head-evaluation/0/DONE | 0 .../0/report.json | 203 ++ .../rla_only-evaluation/0.toml | 41 + .../rla_only-evaluation/0/DONE | 0 .../rla_only-evaluation/0/report.json | 199 ++ .../rla_only-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 208 ++ .../0/DONE | 0 .../0/report.json | 455 ++++ .../all_four_combined-evaluation/0.toml | 74 + .../all_four_combined-evaluation/0/DONE | 0 .../0/report.json | 451 ++++ .../0/DONE | 0 .../0/report.json | 464 ++++ .../0/DONE | 0 .../0/report.json | 202 ++ .../all_minus_cf_fisd-evaluation/0.toml | 63 + .../all_minus_cf_fisd-evaluation/0/DONE | 0 .../0/report.json | 198 ++ .../0/DONE | 0 .../0/report.json | 219 ++ .../0/DONE | 0 .../0/report.json | 445 ++++ .../all_minus_esam-evaluation/0.toml | 64 + .../all_minus_esam-evaluation/0/DONE | 0 .../all_minus_esam-evaluation/0/report.json | 441 ++++ .../0/DONE | 0 .../0/report.json | 457 ++++ .../all_minus_mfb-best-head-evaluation/0/DONE | 0 .../0/report.json | 419 ++++ .../all_minus_mfb-evaluation/0.toml | 61 + .../all_minus_mfb-evaluation/0/DONE | 0 .../all_minus_mfb-evaluation/0/report.json | 415 ++++ .../0/DONE | 0 .../0/report.json | 434 ++++ .../all_minus_rla-best-head-evaluation/0/DONE | 0 .../0/report.json | 446 ++++ .../all_minus_rla-evaluation/0.toml | 67 + .../all_minus_rla-evaluation/0/DONE | 0 .../all_minus_rla-evaluation/0/report.json | 442 ++++ .../0/DONE | 0 .../0/report.json | 461 ++++ .../0/DONE | 0 .../0/report.json | 400 ++++ .../cf_fisd_lambda0.05-evaluation/0.toml | 44 + .../cf_fisd_lambda0.05-evaluation/0/DONE | 0 .../0/report.json | 396 ++++ .../0/DONE | 0 .../0/report.json | 413 ++++ .../0/DONE | 0 .../0/report.json | 400 ++++ .../cf_fisd_lambda0.1-evaluation/0.toml | 44 + .../cf_fisd_lambda0.1-evaluation/0/DONE | 0 .../0/report.json | 396 ++++ .../0/DONE | 0 .../0/report.json | 403 ++++ .../0/DONE | 0 .../0/report.json | 400 ++++ .../cf_fisd_lambda0.2-evaluation/0.toml | 44 + .../cf_fisd_lambda0.2-evaluation/0/DONE | 0 .../0/report.json | 396 ++++ .../0/DONE | 0 .../0/report.json | 409 ++++ .../esam_cf_fisd-best-head-evaluation/0/DONE | 0 .../0/report.json | 410 ++++ .../esam_cf_fisd-evaluation/0.toml | 54 + .../esam_cf_fisd-evaluation/0/DONE | 0 .../esam_cf_fisd-evaluation/0/report.json | 406 ++++ .../0/DONE | 0 .../0/report.json | 413 ++++ .../esam_mfb-best-head-evaluation/0/DONE | 0 .../0/report.json | 193 ++ .../cooking-time/esam_mfb-evaluation/0.toml | 56 + .../cooking-time/esam_mfb-evaluation/0/DONE | 0 .../esam_mfb-evaluation/0/report.json | 189 ++ .../esam_mfb-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 208 ++ .../esam_rho0.001-best-head-evaluation/0/DONE | 0 .../0/report.json | 157 ++ .../esam_rho0.001-evaluation/0.toml | 43 + .../esam_rho0.001-evaluation/0/DONE | 0 .../esam_rho0.001-evaluation/0/report.json | 153 ++ .../0/DONE | 0 .../0/report.json | 168 ++ .../0/DONE | 0 .../0/report.json | 157 ++ .../esam_rho0.0025-evaluation/0.toml | 43 + .../esam_rho0.0025-evaluation/0/DONE | 0 .../esam_rho0.0025-evaluation/0/report.json | 153 ++ .../0/DONE | 0 .../0/report.json | 179 ++ .../esam_rho0.005-best-head-evaluation/0/DONE | 0 .../0/report.json | 157 ++ .../esam_rho0.005-evaluation/0.toml | 43 + .../esam_rho0.005-evaluation/0/DONE | 0 .../esam_rho0.005-evaluation/0/report.json | 153 ++ .../0/DONE | 0 .../0/report.json | 167 ++ .../esam_rho0.01-best-head-evaluation/0/DONE | 0 .../0/report.json | 157 ++ .../esam_rho0.01-evaluation/0.toml | 43 + .../esam_rho0.01-evaluation/0/DONE | 0 .../esam_rho0.01-evaluation/0/report.json | 153 ++ .../0/DONE | 0 .../0/report.json | 172 ++ .../mfb_cf_fisd-best-head-evaluation/0/DONE | 0 .../0/report.json | 436 ++++ .../mfb_cf_fisd-evaluation/0.toml | 57 + .../mfb_cf_fisd-evaluation/0/DONE | 0 .../mfb_cf_fisd-evaluation/0/report.json | 432 ++++ .../0/DONE | 0 .../0/report.json | 449 ++++ .../mfb_keep0.7-best-head-evaluation/0/DONE | 0 .../0/report.json | 183 ++ .../mfb_keep0.7-evaluation/0.toml | 46 + .../mfb_keep0.7-evaluation/0/DONE | 0 .../mfb_keep0.7-evaluation/0/report.json | 179 ++ .../0/DONE | 0 .../0/report.json | 197 ++ .../mfb_keep0.8-best-head-evaluation/0/DONE | 0 .../0/report.json | 183 ++ .../mfb_keep0.8-evaluation/0.toml | 46 + .../mfb_keep0.8-evaluation/0/DONE | 0 .../mfb_keep0.8-evaluation/0/report.json | 179 ++ .../0/DONE | 0 .../0/report.json | 193 ++ .../mfb_keep0.9-best-head-evaluation/0/DONE | 0 .../0/report.json | 183 ++ .../mfb_keep0.9-evaluation/0.toml | 46 + .../mfb_keep0.9-evaluation/0/DONE | 0 .../mfb_keep0.9-evaluation/0/report.json | 179 ++ .../0/DONE | 0 .../0/report.json | 193 ++ .../mfb_keep0.95-best-head-evaluation/0/DONE | 0 .../0/report.json | 183 ++ .../mfb_keep0.95-evaluation/0.toml | 46 + .../mfb_keep0.95-evaluation/0/DONE | 0 .../mfb_keep0.95-evaluation/0/report.json | 179 ++ .../0/DONE | 0 .../0/report.json | 197 ++ .../rla_cf_fisd-best-head-evaluation/0/DONE | 0 .../0/report.json | 409 ++++ .../rla_cf_fisd-evaluation/0.toml | 51 + .../rla_cf_fisd-evaluation/0/DONE | 0 .../rla_cf_fisd-evaluation/0/report.json | 405 ++++ .../0/DONE | 0 .../0/report.json | 413 ++++ .../rla_esam-best-head-evaluation/0/DONE | 0 .../0/report.json | 166 ++ .../cooking-time/rla_esam-evaluation/0.toml | 50 + .../cooking-time/rla_esam-evaluation/0/DONE | 0 .../rla_esam-evaluation/0/report.json | 162 ++ .../rla_esam-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 178 ++ .../rla_mfb-best-head-evaluation/0/DONE | 0 .../0/report.json | 192 ++ .../cooking-time/rla_mfb-evaluation/0.toml | 53 + .../cooking-time/rla_mfb-evaluation/0/DONE | 0 .../rla_mfb-evaluation/0/report.json | 188 ++ .../rla_mfb-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 207 ++ .../0/DONE | 0 .../0/report.json | 156 ++ .../rla_rank2_noise0.0001-evaluation/0.toml | 40 + .../rla_rank2_noise0.0001-evaluation/0/DONE | 0 .../0/report.json | 152 ++ .../0/DONE | 0 .../0/report.json | 165 ++ .../0/DONE | 0 .../0/report.json | 156 ++ .../rla_rank2_noise0.001-evaluation/0.toml | 40 + .../rla_rank2_noise0.001-evaluation/0/DONE | 0 .../0/report.json | 152 ++ .../0/DONE | 0 .../0/report.json | 174 ++ .../0/DONE | 0 .../0/report.json | 156 ++ .../rla_rank2_noise1e-05-evaluation/0.toml | 40 + .../rla_rank2_noise1e-05-evaluation/0/DONE | 0 .../0/report.json | 152 ++ .../0/DONE | 0 .../0/report.json | 175 ++ .../0/DONE | 0 .../0/report.json | 156 ++ .../rla_rank4_noise0.0001-evaluation/0.toml | 40 + .../rla_rank4_noise0.0001-evaluation/0/DONE | 0 .../0/report.json | 152 ++ .../0/DONE | 0 .../0/report.json | 177 ++ .../0/DONE | 0 .../0/report.json | 156 ++ .../rla_rank4_noise0.001-evaluation/0.toml | 40 + .../rla_rank4_noise0.001-evaluation/0/DONE | 0 .../0/report.json | 152 ++ .../0/DONE | 0 .../0/report.json | 178 ++ .../0/DONE | 0 .../0/report.json | 156 ++ .../rla_rank4_noise1e-05-evaluation/0.toml | 40 + .../rla_rank4_noise1e-05-evaluation/0/DONE | 0 .../0/report.json | 152 ++ .../0/DONE | 0 .../0/report.json | 180 ++ .../0/DONE | 0 .../0/report.json | 156 ++ .../rla_rank8_noise0.0001-evaluation/0.toml | 40 + .../rla_rank8_noise0.0001-evaluation/0/DONE | 0 .../0/report.json | 152 ++ .../0/DONE | 0 .../0/report.json | 176 ++ .../0/DONE | 0 .../0/report.json | 156 ++ .../rla_rank8_noise0.001-evaluation/0.toml | 40 + .../rla_rank8_noise0.001-evaluation/0/DONE | 0 .../0/report.json | 152 ++ .../0/DONE | 0 .../0/report.json | 164 ++ .../0/DONE | 0 .../0/report.json | 156 ++ .../rla_rank8_noise1e-05-evaluation/0.toml | 40 + .../rla_rank8_noise1e-05-evaluation/0/DONE | 0 .../0/report.json | 152 ++ .../0/DONE | 0 .../0/report.json | 170 ++ .../0/DONE | 0 .../0/report.json | 506 +++++ .../all_four_combined-evaluation/0.toml | 74 + .../all_four_combined-evaluation/0/DONE | 0 .../0/report.json | 502 +++++ .../0/DONE | 0 .../0/report.json | 510 +++++ .../0/DONE | 0 .../0/report.json | 225 ++ .../all_minus_cf_fisd-evaluation/0.toml | 63 + .../all_minus_cf_fisd-evaluation/0/DONE | 0 .../0/report.json | 221 ++ .../0/DONE | 0 .../0/report.json | 236 +++ .../0/DONE | 0 .../0/report.json | 496 +++++ .../all_minus_esam-evaluation/0.toml | 64 + .../all_minus_esam-evaluation/0/DONE | 0 .../all_minus_esam-evaluation/0/report.json | 492 +++++ .../0/DONE | 0 .../0/report.json | 502 +++++ .../all_minus_mfb-best-head-evaluation/0/DONE | 0 .../0/report.json | 470 +++++ .../all_minus_mfb-evaluation/0.toml | 61 + .../all_minus_mfb-evaluation/0/DONE | 0 .../all_minus_mfb-evaluation/0/report.json | 466 ++++ .../0/DONE | 0 .../0/report.json | 475 +++++ .../all_minus_rla-best-head-evaluation/0/DONE | 0 .../0/report.json | 495 +++++ .../all_minus_rla-evaluation/0.toml | 67 + .../all_minus_rla-evaluation/0/DONE | 0 .../all_minus_rla-evaluation/0/report.json | 491 +++++ .../0/DONE | 0 .../0/report.json | 508 +++++ .../0/DONE | 0 .../0/report.json | 449 ++++ .../cf_fisd_lambda0.05-evaluation/0.toml | 44 + .../cf_fisd_lambda0.05-evaluation/0/DONE | 0 .../0/report.json | 445 ++++ .../0/DONE | 0 .../0/report.json | 457 ++++ .../0/DONE | 0 .../0/report.json | 449 ++++ .../cf_fisd_lambda0.1-evaluation/0.toml | 44 + .../cf_fisd_lambda0.1-evaluation/0/DONE | 0 .../0/report.json | 445 ++++ .../0/DONE | 0 .../0/report.json | 469 +++++ .../0/DONE | 0 .../0/report.json | 449 ++++ .../cf_fisd_lambda0.2-evaluation/0.toml | 44 + .../cf_fisd_lambda0.2-evaluation/0/DONE | 0 .../0/report.json | 445 ++++ .../0/DONE | 0 .../0/report.json | 455 ++++ .../esam_cf_fisd-best-head-evaluation/0/DONE | 0 .../0/report.json | 459 ++++ .../esam_cf_fisd-evaluation/0.toml | 54 + .../esam_cf_fisd-evaluation/0/DONE | 0 .../esam_cf_fisd-evaluation/0/report.json | 455 ++++ .../0/DONE | 0 .../0/report.json | 469 +++++ .../esam_mfb-best-head-evaluation/0/DONE | 0 .../0/report.json | 214 ++ .../delivery-eta/esam_mfb-evaluation/0.toml | 56 + .../delivery-eta/esam_mfb-evaluation/0/DONE | 0 .../esam_mfb-evaluation/0/report.json | 210 ++ .../esam_mfb-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 233 ++ .../esam_rho0.001-best-head-evaluation/0/DONE | 0 .../0/report.json | 178 ++ .../esam_rho0.001-evaluation/0.toml | 43 + .../esam_rho0.001-evaluation/0/DONE | 0 .../esam_rho0.001-evaluation/0/report.json | 174 ++ .../0/DONE | 0 .../0/report.json | 187 ++ .../0/DONE | 0 .../0/report.json | 178 ++ .../esam_rho0.0025-evaluation/0.toml | 43 + .../esam_rho0.0025-evaluation/0/DONE | 0 .../esam_rho0.0025-evaluation/0/report.json | 174 ++ .../0/DONE | 0 .../0/report.json | 194 ++ .../esam_rho0.005-best-head-evaluation/0/DONE | 0 .../0/report.json | 178 ++ .../esam_rho0.005-evaluation/0.toml | 43 + .../esam_rho0.005-evaluation/0/DONE | 0 .../esam_rho0.005-evaluation/0/report.json | 174 ++ .../0/DONE | 0 .../0/report.json | 190 ++ .../esam_rho0.01-best-head-evaluation/0/DONE | 0 .../0/report.json | 178 ++ .../esam_rho0.01-evaluation/0.toml | 43 + .../esam_rho0.01-evaluation/0/DONE | 0 .../esam_rho0.01-evaluation/0/report.json | 174 ++ .../0/DONE | 0 .../0/report.json | 199 ++ .../mfb_cf_fisd-best-head-evaluation/0/DONE | 0 .../0/report.json | 485 +++++ .../mfb_cf_fisd-evaluation/0.toml | 57 + .../mfb_cf_fisd-evaluation/0/DONE | 0 .../mfb_cf_fisd-evaluation/0/report.json | 481 +++++ .../0/DONE | 0 .../0/report.json | 494 +++++ .../mfb_keep0.7-best-head-evaluation/0/DONE | 0 .../0/report.json | 204 ++ .../mfb_keep0.7-evaluation/0.toml | 46 + .../mfb_keep0.7-evaluation/0/DONE | 0 .../mfb_keep0.7-evaluation/0/report.json | 200 ++ .../0/DONE | 0 .../0/report.json | 211 ++ .../mfb_keep0.8-best-head-evaluation/0/DONE | 0 .../0/report.json | 204 ++ .../mfb_keep0.8-evaluation/0.toml | 46 + .../mfb_keep0.8-evaluation/0/DONE | 0 .../mfb_keep0.8-evaluation/0/report.json | 200 ++ .../0/DONE | 0 .../0/report.json | 220 ++ .../mfb_keep0.9-best-head-evaluation/0/DONE | 0 .../0/report.json | 204 ++ .../mfb_keep0.9-evaluation/0.toml | 46 + .../mfb_keep0.9-evaluation/0/DONE | 0 .../mfb_keep0.9-evaluation/0/report.json | 200 ++ .../0/DONE | 0 .../0/report.json | 217 ++ .../mfb_keep0.95-best-head-evaluation/0/DONE | 0 .../0/report.json | 204 ++ .../mfb_keep0.95-evaluation/0.toml | 46 + .../mfb_keep0.95-evaluation/0/DONE | 0 .../mfb_keep0.95-evaluation/0/report.json | 200 ++ .../0/DONE | 0 .../0/report.json | 213 ++ .../rla_cf_fisd-best-head-evaluation/0/DONE | 0 .../0/report.json | 460 ++++ .../rla_cf_fisd-evaluation/0.toml | 51 + .../rla_cf_fisd-evaluation/0/DONE | 0 .../rla_cf_fisd-evaluation/0/report.json | 456 ++++ .../0/DONE | 0 .../0/report.json | 468 +++++ .../rla_esam-best-head-evaluation/0/DONE | 0 .../0/report.json | 189 ++ .../delivery-eta/rla_esam-evaluation/0.toml | 50 + .../delivery-eta/rla_esam-evaluation/0/DONE | 0 .../rla_esam-evaluation/0/report.json | 185 ++ .../rla_esam-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 196 ++ .../rla_mfb-best-head-evaluation/0/DONE | 0 .../0/report.json | 215 ++ .../delivery-eta/rla_mfb-evaluation/0.toml | 53 + .../delivery-eta/rla_mfb-evaluation/0/DONE | 0 .../rla_mfb-evaluation/0/report.json | 211 ++ .../rla_mfb-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 221 ++ .../0/DONE | 0 .../0/report.json | 179 ++ .../rla_rank2_noise0.0001-evaluation/0.toml | 40 + .../rla_rank2_noise0.0001-evaluation/0/DONE | 0 .../0/report.json | 175 ++ .../0/DONE | 0 .../0/report.json | 188 ++ .../0/DONE | 0 .../0/report.json | 179 ++ .../rla_rank2_noise0.001-evaluation/0.toml | 40 + .../rla_rank2_noise0.001-evaluation/0/DONE | 0 .../0/report.json | 175 ++ .../0/DONE | 0 .../0/report.json | 194 ++ .../0/DONE | 0 .../0/report.json | 179 ++ .../rla_rank2_noise1e-05-evaluation/0.toml | 40 + .../rla_rank2_noise1e-05-evaluation/0/DONE | 0 .../0/report.json | 175 ++ .../0/DONE | 0 .../0/report.json | 194 ++ .../0/DONE | 0 .../0/report.json | 179 ++ .../rla_rank4_noise0.0001-evaluation/0.toml | 40 + .../rla_rank4_noise0.0001-evaluation/0/DONE | 0 .../0/report.json | 175 ++ .../0/DONE | 0 .../0/report.json | 184 ++ .../0/DONE | 0 .../0/report.json | 179 ++ .../rla_rank4_noise0.001-evaluation/0.toml | 40 + .../rla_rank4_noise0.001-evaluation/0/DONE | 0 .../0/report.json | 175 ++ .../0/DONE | 0 .../0/report.json | 190 ++ .../0/DONE | 0 .../0/report.json | 179 ++ .../rla_rank4_noise1e-05-evaluation/0.toml | 40 + .../rla_rank4_noise1e-05-evaluation/0/DONE | 0 .../0/report.json | 175 ++ .../0/DONE | 0 .../0/report.json | 187 ++ .../0/DONE | 0 .../0/report.json | 179 ++ .../rla_rank8_noise0.0001-evaluation/0.toml | 40 + .../rla_rank8_noise0.0001-evaluation/0/DONE | 0 .../0/report.json | 175 ++ .../0/DONE | 0 .../0/report.json | 185 ++ .../0/DONE | 0 .../0/report.json | 179 ++ .../rla_rank8_noise0.001-evaluation/0.toml | 40 + .../rla_rank8_noise0.001-evaluation/0/DONE | 0 .../0/report.json | 175 ++ .../0/DONE | 0 .../0/report.json | 186 ++ .../0/DONE | 0 .../0/report.json | 179 ++ .../rla_rank8_noise1e-05-evaluation/0.toml | 40 + .../rla_rank8_noise1e-05-evaluation/0/DONE | 0 .../0/report.json | 175 ++ .../0/DONE | 0 .../0/report.json | 187 ++ .../0/DONE | 0 .../0/report.json | 445 ++++ .../all_four_combined-evaluation/0.toml | 74 + .../all_four_combined-evaluation/0/DONE | 0 .../0/report.json | 441 ++++ .../0/DONE | 0 .../0/report.json | 447 ++++ .../0/DONE | 0 .../0/report.json | 274 +++ .../all_minus_cf_fisd-evaluation/0.toml | 63 + .../all_minus_cf_fisd-evaluation/0/DONE | 0 .../0/report.json | 270 +++ .../0/DONE | 0 .../0/report.json | 275 +++ .../0/DONE | 0 .../0/report.json | 435 ++++ .../all_minus_esam-evaluation/0.toml | 64 + .../all_minus_esam-evaluation/0/DONE | 0 .../all_minus_esam-evaluation/0/report.json | 431 ++++ .../0/DONE | 0 .../0/report.json | 436 ++++ .../all_minus_mfb-best-head-evaluation/0/DONE | 0 .../0/report.json | 409 ++++ .../all_minus_mfb-evaluation/0.toml | 61 + .../all_minus_mfb-evaluation/0/DONE | 0 .../all_minus_mfb-evaluation/0/report.json | 405 ++++ .../0/DONE | 0 .../0/report.json | 411 ++++ .../all_minus_rla-best-head-evaluation/0/DONE | 0 .../0/report.json | 436 ++++ .../all_minus_rla-evaluation/0.toml | 67 + .../all_minus_rla-evaluation/0/DONE | 0 .../all_minus_rla-evaluation/0/report.json | 432 ++++ .../0/DONE | 0 .../0/report.json | 438 ++++ .../0/DONE | 0 .../0/report.json | 390 ++++ .../cf_fisd_lambda0.05-evaluation/0.toml | 44 + .../cf_fisd_lambda0.05-evaluation/0/DONE | 0 .../0/report.json | 386 ++++ .../0/DONE | 0 .../0/report.json | 391 ++++ .../0/DONE | 0 .../0/report.json | 390 ++++ .../cf_fisd_lambda0.1-evaluation/0.toml | 44 + .../cf_fisd_lambda0.1-evaluation/0/DONE | 0 .../0/report.json | 386 ++++ .../0/DONE | 0 .../0/report.json | 391 ++++ .../0/DONE | 0 .../0/report.json | 390 ++++ .../cf_fisd_lambda0.2-evaluation/0.toml | 44 + .../cf_fisd_lambda0.2-evaluation/0/DONE | 0 .../0/report.json | 386 ++++ .../0/DONE | 0 .../0/report.json | 392 ++++ .../esam_cf_fisd-best-head-evaluation/0/DONE | 0 .../0/report.json | 400 ++++ .../esam_cf_fisd-evaluation/0.toml | 54 + .../esam_cf_fisd-evaluation/0/DONE | 0 .../esam_cf_fisd-evaluation/0/report.json | 396 ++++ .../0/DONE | 0 .../0/report.json | 401 ++++ .../esam_mfb-best-head-evaluation/0/DONE | 0 .../0/report.json | 265 +++ .../ecom-offers/esam_mfb-evaluation/0.toml | 56 + .../ecom-offers/esam_mfb-evaluation/0/DONE | 0 .../esam_mfb-evaluation/0/report.json | 261 +++ .../esam_mfb-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 266 +++ .../esam_rho0.001-best-head-evaluation/0/DONE | 0 .../0/report.json | 229 ++ .../esam_rho0.001-evaluation/0.toml | 43 + .../esam_rho0.001-evaluation/0/DONE | 0 .../esam_rho0.001-evaluation/0/report.json | 225 ++ .../0/DONE | 0 .../0/report.json | 230 ++ .../0/DONE | 0 .../0/report.json | 229 ++ .../esam_rho0.0025-evaluation/0.toml | 43 + .../esam_rho0.0025-evaluation/0/DONE | 0 .../esam_rho0.0025-evaluation/0/report.json | 225 ++ .../0/DONE | 0 .../0/report.json | 230 ++ .../esam_rho0.005-best-head-evaluation/0/DONE | 0 .../0/report.json | 229 ++ .../esam_rho0.005-evaluation/0.toml | 43 + .../esam_rho0.005-evaluation/0/DONE | 0 .../esam_rho0.005-evaluation/0/report.json | 225 ++ .../0/DONE | 0 .../0/report.json | 230 ++ .../esam_rho0.01-best-head-evaluation/0/DONE | 0 .../0/report.json | 229 ++ .../esam_rho0.01-evaluation/0.toml | 43 + .../esam_rho0.01-evaluation/0/DONE | 0 .../esam_rho0.01-evaluation/0/report.json | 225 ++ .../0/DONE | 0 .../0/report.json | 230 ++ .../mfb_cf_fisd-best-head-evaluation/0/DONE | 0 .../0/report.json | 426 ++++ .../ecom-offers/mfb_cf_fisd-evaluation/0.toml | 57 + .../ecom-offers/mfb_cf_fisd-evaluation/0/DONE | 0 .../mfb_cf_fisd-evaluation/0/report.json | 422 ++++ .../0/DONE | 0 .../0/report.json | 428 ++++ .../mfb_keep0.7-best-head-evaluation/0/DONE | 0 .../0/report.json | 255 +++ .../ecom-offers/mfb_keep0.7-evaluation/0.toml | 46 + .../ecom-offers/mfb_keep0.7-evaluation/0/DONE | 0 .../mfb_keep0.7-evaluation/0/report.json | 251 +++ .../0/DONE | 0 .../0/report.json | 255 +++ .../mfb_keep0.8-best-head-evaluation/0/DONE | 0 .../0/report.json | 255 +++ .../ecom-offers/mfb_keep0.8-evaluation/0.toml | 46 + .../ecom-offers/mfb_keep0.8-evaluation/0/DONE | 0 .../mfb_keep0.8-evaluation/0/report.json | 251 +++ .../0/DONE | 0 .../0/report.json | 256 +++ .../mfb_keep0.9-best-head-evaluation/0/DONE | 0 .../0/report.json | 255 +++ .../ecom-offers/mfb_keep0.9-evaluation/0.toml | 46 + .../ecom-offers/mfb_keep0.9-evaluation/0/DONE | 0 .../mfb_keep0.9-evaluation/0/report.json | 251 +++ .../0/DONE | 0 .../0/report.json | 256 +++ .../mfb_keep0.95-best-head-evaluation/0/DONE | 0 .../0/report.json | 255 +++ .../mfb_keep0.95-evaluation/0.toml | 46 + .../mfb_keep0.95-evaluation/0/DONE | 0 .../mfb_keep0.95-evaluation/0/report.json | 251 +++ .../0/DONE | 0 .../0/report.json | 256 +++ .../rla_cf_fisd-best-head-evaluation/0/DONE | 0 .../0/report.json | 399 ++++ .../ecom-offers/rla_cf_fisd-evaluation/0.toml | 51 + .../ecom-offers/rla_cf_fisd-evaluation/0/DONE | 0 .../rla_cf_fisd-evaluation/0/report.json | 395 ++++ .../0/DONE | 0 .../0/report.json | 401 ++++ .../rla_esam-best-head-evaluation/0/DONE | 0 .../0/report.json | 238 +++ .../ecom-offers/rla_esam-evaluation/0.toml | 50 + .../ecom-offers/rla_esam-evaluation/0/DONE | 0 .../rla_esam-evaluation/0/report.json | 234 +++ .../rla_esam-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 238 +++ .../rla_mfb-best-head-evaluation/0/DONE | 0 .../0/report.json | 264 +++ .../ecom-offers/rla_mfb-evaluation/0.toml | 53 + .../ecom-offers/rla_mfb-evaluation/0/DONE | 0 .../rla_mfb-evaluation/0/report.json | 260 +++ .../rla_mfb-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 265 +++ .../0/DONE | 0 .../0/report.json | 228 ++ .../rla_rank2_noise0.0001-evaluation/0.toml | 40 + .../rla_rank2_noise0.0001-evaluation/0/DONE | 0 .../0/report.json | 224 ++ .../0/DONE | 0 .../0/report.json | 228 ++ .../0/DONE | 0 .../0/report.json | 228 ++ .../rla_rank2_noise0.001-evaluation/0.toml | 40 + .../rla_rank2_noise0.001-evaluation/0/DONE | 0 .../0/report.json | 224 ++ .../0/DONE | 0 .../0/report.json | 228 ++ .../0/DONE | 0 .../0/report.json | 228 ++ .../rla_rank2_noise1e-05-evaluation/0.toml | 40 + .../rla_rank2_noise1e-05-evaluation/0/DONE | 0 .../0/report.json | 224 ++ .../0/DONE | 0 .../0/report.json | 228 ++ .../0/DONE | 0 .../0/report.json | 228 ++ .../rla_rank4_noise0.0001-evaluation/0.toml | 40 + .../rla_rank4_noise0.0001-evaluation/0/DONE | 0 .../0/report.json | 224 ++ .../0/DONE | 0 .../0/report.json | 228 ++ .../0/DONE | 0 .../0/report.json | 228 ++ .../rla_rank4_noise0.001-evaluation/0.toml | 40 + .../rla_rank4_noise0.001-evaluation/0/DONE | 0 .../0/report.json | 224 ++ .../0/DONE | 0 .../0/report.json | 228 ++ .../0/DONE | 0 .../0/report.json | 228 ++ .../rla_rank4_noise1e-05-evaluation/0.toml | 40 + .../rla_rank4_noise1e-05-evaluation/0/DONE | 0 .../0/report.json | 224 ++ .../0/DONE | 0 .../0/report.json | 228 ++ .../0/DONE | 0 .../0/report.json | 228 ++ .../rla_rank8_noise0.0001-evaluation/0.toml | 40 + .../rla_rank8_noise0.0001-evaluation/0/DONE | 0 .../0/report.json | 224 ++ .../0/DONE | 0 .../0/report.json | 228 ++ .../0/DONE | 0 .../0/report.json | 228 ++ .../rla_rank8_noise0.001-evaluation/0.toml | 40 + .../rla_rank8_noise0.001-evaluation/0/DONE | 0 .../0/report.json | 224 ++ .../0/DONE | 0 .../0/report.json | 228 ++ .../0/DONE | 0 .../0/report.json | 228 ++ .../rla_rank8_noise1e-05-evaluation/0.toml | 40 + .../rla_rank8_noise1e-05-evaluation/0/DONE | 0 .../0/report.json | 224 ++ .../0/DONE | 0 .../0/report.json | 228 ++ .../0/DONE | 0 .../0/report.json | 680 ++++++ .../all_four_combined-evaluation/0.toml | 75 + .../all_four_combined-evaluation/0/DONE | 0 .../0/report.json | 676 ++++++ .../0/DONE | 0 .../0/report.json | 690 ++++++ .../0/DONE | 0 .../0/report.json | 321 +++ .../all_minus_cf_fisd-evaluation/0.toml | 64 + .../all_minus_cf_fisd-evaluation/0/DONE | 0 .../0/report.json | 317 +++ .../0/DONE | 0 .../0/report.json | 333 +++ .../0/DONE | 0 .../0/report.json | 670 ++++++ .../all_minus_esam-evaluation/0.toml | 65 + .../all_minus_esam-evaluation/0/DONE | 0 .../all_minus_esam-evaluation/0/report.json | 666 ++++++ .../0/DONE | 0 .../0/report.json | 678 ++++++ .../all_minus_mfb-best-head-evaluation/0/DONE | 0 .../0/report.json | 644 ++++++ .../all_minus_mfb-evaluation/0.toml | 62 + .../all_minus_mfb-evaluation/0/DONE | 0 .../all_minus_mfb-evaluation/0/report.json | 640 ++++++ .../0/DONE | 0 .../0/report.json | 651 ++++++ .../all_minus_rla-best-head-evaluation/0/DONE | 0 .../0/report.json | 667 ++++++ .../all_minus_rla-evaluation/0.toml | 68 + .../all_minus_rla-evaluation/0/DONE | 0 .../all_minus_rla-evaluation/0/report.json | 663 ++++++ .../0/DONE | 0 .../0/report.json | 675 ++++++ .../0/DONE | 0 .../0/report.json | 621 ++++++ .../cf_fisd_lambda0.05-evaluation/0.toml | 45 + .../cf_fisd_lambda0.05-evaluation/0/DONE | 0 .../0/report.json | 617 ++++++ .../0/DONE | 0 .../0/report.json | 634 ++++++ .../0/DONE | 0 .../0/report.json | 621 ++++++ .../cf_fisd_lambda0.1-evaluation/0.toml | 45 + .../cf_fisd_lambda0.1-evaluation/0/DONE | 0 .../0/report.json | 617 ++++++ .../0/DONE | 0 .../0/report.json | 639 ++++++ .../0/DONE | 0 .../0/report.json | 621 ++++++ .../cf_fisd_lambda0.2-evaluation/0.toml | 45 + .../cf_fisd_lambda0.2-evaluation/0/DONE | 0 .../0/report.json | 617 ++++++ .../0/DONE | 0 .../0/report.json | 634 ++++++ .../esam_cf_fisd-best-head-evaluation/0/DONE | 0 .../0/report.json | 631 ++++++ .../esam_cf_fisd-evaluation/0.toml | 55 + .../esam_cf_fisd-evaluation/0/DONE | 0 .../esam_cf_fisd-evaluation/0/report.json | 627 ++++++ .../0/DONE | 0 .../0/report.json | 642 ++++++ .../esam_mfb-best-head-evaluation/0/DONE | 0 .../0/report.json | 308 +++ .../esam_mfb-evaluation/0.toml | 57 + .../esam_mfb-evaluation/0/DONE | 0 .../esam_mfb-evaluation/0/report.json | 304 +++ .../esam_mfb-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 317 +++ .../esam_rho0.001-best-head-evaluation/0/DONE | 0 .../0/report.json | 272 +++ .../esam_rho0.001-evaluation/0.toml | 44 + .../esam_rho0.001-evaluation/0/DONE | 0 .../esam_rho0.001-evaluation/0/report.json | 268 +++ .../0/DONE | 0 .../0/report.json | 285 +++ .../0/DONE | 0 .../0/report.json | 272 +++ .../esam_rho0.0025-evaluation/0.toml | 44 + .../esam_rho0.0025-evaluation/0/DONE | 0 .../esam_rho0.0025-evaluation/0/report.json | 268 +++ .../0/DONE | 0 .../0/report.json | 281 +++ .../esam_rho0.005-best-head-evaluation/0/DONE | 0 .../0/report.json | 272 +++ .../esam_rho0.005-evaluation/0.toml | 44 + .../esam_rho0.005-evaluation/0/DONE | 0 .../esam_rho0.005-evaluation/0/report.json | 268 +++ .../0/DONE | 0 .../0/report.json | 290 +++ .../esam_rho0.01-best-head-evaluation/0/DONE | 0 .../0/report.json | 272 +++ .../esam_rho0.01-evaluation/0.toml | 44 + .../esam_rho0.01-evaluation/0/DONE | 0 .../esam_rho0.01-evaluation/0/report.json | 268 +++ .../0/DONE | 0 .../0/report.json | 282 +++ .../mfb_cf_fisd-best-head-evaluation/0/DONE | 0 .../0/report.json | 657 ++++++ .../mfb_cf_fisd-evaluation/0.toml | 58 + .../mfb_cf_fisd-evaluation/0/DONE | 0 .../mfb_cf_fisd-evaluation/0/report.json | 653 ++++++ .../0/DONE | 0 .../0/report.json | 666 ++++++ .../mfb_keep0.7-best-head-evaluation/0/DONE | 0 .../0/report.json | 298 +++ .../mfb_keep0.7-evaluation/0.toml | 47 + .../mfb_keep0.7-evaluation/0/DONE | 0 .../mfb_keep0.7-evaluation/0/report.json | 294 +++ .../0/DONE | 0 .../0/report.json | 305 +++ .../mfb_keep0.8-best-head-evaluation/0/DONE | 0 .../0/report.json | 298 +++ .../mfb_keep0.8-evaluation/0.toml | 47 + .../mfb_keep0.8-evaluation/0/DONE | 0 .../mfb_keep0.8-evaluation/0/report.json | 294 +++ .../0/DONE | 0 .../0/report.json | 305 +++ .../mfb_keep0.9-best-head-evaluation/0/DONE | 0 .../0/report.json | 298 +++ .../mfb_keep0.9-evaluation/0.toml | 47 + .../mfb_keep0.9-evaluation/0/DONE | 0 .../mfb_keep0.9-evaluation/0/report.json | 294 +++ .../0/DONE | 0 .../0/report.json | 307 +++ .../mfb_keep0.95-best-head-evaluation/0/DONE | 0 .../0/report.json | 298 +++ .../mfb_keep0.95-evaluation/0.toml | 47 + .../mfb_keep0.95-evaluation/0/DONE | 0 .../mfb_keep0.95-evaluation/0/report.json | 294 +++ .../0/DONE | 0 .../0/report.json | 304 +++ .../rla_cf_fisd-best-head-evaluation/0/DONE | 0 .../0/report.json | 634 ++++++ .../rla_cf_fisd-evaluation/0.toml | 52 + .../rla_cf_fisd-evaluation/0/DONE | 0 .../rla_cf_fisd-evaluation/0/report.json | 630 ++++++ .../0/DONE | 0 .../0/report.json | 637 ++++++ .../rla_esam-best-head-evaluation/0/DONE | 0 .../0/report.json | 285 +++ .../rla_esam-evaluation/0.toml | 51 + .../rla_esam-evaluation/0/DONE | 0 .../rla_esam-evaluation/0/report.json | 281 +++ .../rla_esam-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 295 +++ .../rla_mfb-best-head-evaluation/0/DONE | 0 .../0/report.json | 311 +++ .../rla_mfb-evaluation/0.toml | 54 + .../rla_mfb-evaluation/0/DONE | 0 .../rla_mfb-evaluation/0/report.json | 307 +++ .../rla_mfb-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 320 +++ .../0/DONE | 0 .../0/report.json | 275 +++ .../rla_rank2_noise0.0001-evaluation/0.toml | 41 + .../rla_rank2_noise0.0001-evaluation/0/DONE | 0 .../0/report.json | 271 +++ .../0/DONE | 0 .../0/report.json | 286 +++ .../0/DONE | 0 .../0/report.json | 275 +++ .../rla_rank2_noise0.001-evaluation/0.toml | 41 + .../rla_rank2_noise0.001-evaluation/0/DONE | 0 .../0/report.json | 271 +++ .../0/DONE | 0 .../0/report.json | 283 +++ .../0/DONE | 0 .../0/report.json | 275 +++ .../rla_rank2_noise1e-05-evaluation/0.toml | 41 + .../rla_rank2_noise1e-05-evaluation/0/DONE | 0 .../0/report.json | 271 +++ .../0/DONE | 0 .../0/report.json | 292 +++ .../0/DONE | 0 .../0/report.json | 275 +++ .../rla_rank4_noise0.0001-evaluation/0.toml | 41 + .../rla_rank4_noise0.0001-evaluation/0/DONE | 0 .../0/report.json | 271 +++ .../0/DONE | 0 .../0/report.json | 285 +++ .../0/DONE | 0 .../0/report.json | 275 +++ .../rla_rank4_noise0.001-evaluation/0.toml | 41 + .../rla_rank4_noise0.001-evaluation/0/DONE | 0 .../0/report.json | 271 +++ .../0/DONE | 0 .../0/report.json | 278 +++ .../0/DONE | 0 .../0/report.json | 275 +++ .../rla_rank4_noise1e-05-evaluation/0.toml | 41 + .../rla_rank4_noise1e-05-evaluation/0/DONE | 0 .../0/report.json | 271 +++ .../0/DONE | 0 .../0/report.json | 284 +++ .../0/DONE | 0 .../0/report.json | 275 +++ .../rla_rank8_noise0.0001-evaluation/0.toml | 41 + .../rla_rank8_noise0.0001-evaluation/0/DONE | 0 .../0/report.json | 271 +++ .../0/DONE | 0 .../0/report.json | 280 +++ .../0/DONE | 0 .../0/report.json | 275 +++ .../rla_rank8_noise0.001-evaluation/0.toml | 41 + .../rla_rank8_noise0.001-evaluation/0/DONE | 0 .../0/report.json | 271 +++ .../0/DONE | 0 .../0/report.json | 282 +++ .../0/DONE | 0 .../0/report.json | 275 +++ .../rla_rank8_noise1e-05-evaluation/0.toml | 41 + .../rla_rank8_noise1e-05-evaluation/0/DONE | 0 .../0/report.json | 271 +++ .../0/DONE | 0 .../0/report.json | 287 +++ .../0/DONE | 0 .../0/report.json | 702 +++++++ .../all_four_combined-evaluation/0.toml | 75 + .../all_four_combined-evaluation/0/DONE | 0 .../0/report.json | 698 ++++++ .../0/DONE | 0 .../0/report.json | 710 +++++++ .../0/DONE | 0 .../0/report.json | 249 +++ .../all_minus_cf_fisd-evaluation/0.toml | 64 + .../all_minus_cf_fisd-evaluation/0/DONE | 0 .../0/report.json | 245 +++ .../0/DONE | 0 .../0/report.json | 256 +++ .../0/DONE | 0 .../0/report.json | 692 ++++++ .../all_minus_esam-evaluation/0.toml | 65 + .../all_minus_esam-evaluation/0/DONE | 0 .../all_minus_esam-evaluation/0/report.json | 688 ++++++ .../0/DONE | 0 .../0/report.json | 697 ++++++ .../all_minus_mfb-best-head-evaluation/0/DONE | 0 .../0/report.json | 666 ++++++ .../all_minus_mfb-evaluation/0.toml | 62 + .../all_minus_mfb-evaluation/0/DONE | 0 .../all_minus_mfb-evaluation/0/report.json | 662 ++++++ .../0/DONE | 0 .../0/report.json | 673 ++++++ .../all_minus_rla-best-head-evaluation/0/DONE | 0 .../0/report.json | 689 ++++++ .../all_minus_rla-evaluation/0.toml | 68 + .../all_minus_rla-evaluation/0/DONE | 0 .../all_minus_rla-evaluation/0/report.json | 685 ++++++ .../0/DONE | 0 .../0/report.json | 694 ++++++ .../0/DONE | 0 .../0/report.json | 643 ++++++ .../cf_fisd_lambda0.05-evaluation/0.toml | 45 + .../cf_fisd_lambda0.05-evaluation/0/DONE | 0 .../0/report.json | 639 ++++++ .../0/DONE | 0 .../0/report.json | 647 ++++++ .../0/DONE | 0 .../0/report.json | 643 ++++++ .../cf_fisd_lambda0.1-evaluation/0.toml | 45 + .../cf_fisd_lambda0.1-evaluation/0/DONE | 0 .../0/report.json | 639 ++++++ .../0/DONE | 0 .../0/report.json | 649 ++++++ .../0/DONE | 0 .../0/report.json | 643 ++++++ .../cf_fisd_lambda0.2-evaluation/0.toml | 45 + .../cf_fisd_lambda0.2-evaluation/0/DONE | 0 .../0/report.json | 639 ++++++ .../0/DONE | 0 .../0/report.json | 654 ++++++ .../esam_cf_fisd-best-head-evaluation/0/DONE | 0 .../0/report.json | 653 ++++++ .../esam_cf_fisd-evaluation/0.toml | 55 + .../esam_cf_fisd-evaluation/0/DONE | 0 .../esam_cf_fisd-evaluation/0/report.json | 649 ++++++ .../0/DONE | 0 .../0/report.json | 662 ++++++ .../esam_mfb-best-head-evaluation/0/DONE | 0 .../0/report.json | 236 +++ .../esam_mfb-evaluation/0.toml | 57 + .../esam_mfb-evaluation/0/DONE | 0 .../esam_mfb-evaluation/0/report.json | 232 ++ .../esam_mfb-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 246 +++ .../esam_rho0.001-best-head-evaluation/0/DONE | 0 .../0/report.json | 200 ++ .../esam_rho0.001-evaluation/0.toml | 44 + .../esam_rho0.001-evaluation/0/DONE | 0 .../esam_rho0.001-evaluation/0/report.json | 196 ++ .../0/DONE | 0 .../0/report.json | 212 ++ .../0/DONE | 0 .../0/report.json | 200 ++ .../esam_rho0.0025-evaluation/0.toml | 44 + .../esam_rho0.0025-evaluation/0/DONE | 0 .../esam_rho0.0025-evaluation/0/report.json | 196 ++ .../0/DONE | 0 .../0/report.json | 207 ++ .../esam_rho0.005-best-head-evaluation/0/DONE | 0 .../0/report.json | 200 ++ .../esam_rho0.005-evaluation/0.toml | 44 + .../esam_rho0.005-evaluation/0/DONE | 0 .../esam_rho0.005-evaluation/0/report.json | 196 ++ .../0/DONE | 0 .../0/report.json | 209 ++ .../esam_rho0.01-best-head-evaluation/0/DONE | 0 .../0/report.json | 200 ++ .../esam_rho0.01-evaluation/0.toml | 44 + .../esam_rho0.01-evaluation/0/DONE | 0 .../esam_rho0.01-evaluation/0/report.json | 196 ++ .../0/DONE | 0 .../0/report.json | 206 ++ .../mfb_cf_fisd-best-head-evaluation/0/DONE | 0 .../0/report.json | 679 ++++++ .../mfb_cf_fisd-evaluation/0.toml | 58 + .../mfb_cf_fisd-evaluation/0/DONE | 0 .../mfb_cf_fisd-evaluation/0/report.json | 675 ++++++ .../0/DONE | 0 .../0/report.json | 688 ++++++ .../mfb_keep0.7-best-head-evaluation/0/DONE | 0 .../0/report.json | 226 ++ .../mfb_keep0.7-evaluation/0.toml | 47 + .../mfb_keep0.7-evaluation/0/DONE | 0 .../mfb_keep0.7-evaluation/0/report.json | 222 ++ .../0/DONE | 0 .../0/report.json | 236 +++ .../mfb_keep0.8-best-head-evaluation/0/DONE | 0 .../0/report.json | 226 ++ .../mfb_keep0.8-evaluation/0.toml | 47 + .../mfb_keep0.8-evaluation/0/DONE | 0 .../mfb_keep0.8-evaluation/0/report.json | 222 ++ .../0/DONE | 0 .../0/report.json | 235 +++ .../mfb_keep0.9-best-head-evaluation/0/DONE | 0 .../0/report.json | 226 ++ .../mfb_keep0.9-evaluation/0.toml | 47 + .../mfb_keep0.9-evaluation/0/DONE | 0 .../mfb_keep0.9-evaluation/0/report.json | 222 ++ .../0/DONE | 0 .../0/report.json | 234 +++ .../mfb_keep0.95-best-head-evaluation/0/DONE | 0 .../0/report.json | 226 ++ .../mfb_keep0.95-evaluation/0.toml | 47 + .../mfb_keep0.95-evaluation/0/DONE | 0 .../mfb_keep0.95-evaluation/0/report.json | 222 ++ .../0/DONE | 0 .../0/report.json | 235 +++ .../rla_cf_fisd-best-head-evaluation/0/DONE | 0 .../0/report.json | 656 ++++++ .../rla_cf_fisd-evaluation/0.toml | 52 + .../rla_cf_fisd-evaluation/0/DONE | 0 .../rla_cf_fisd-evaluation/0/report.json | 652 ++++++ .../0/DONE | 0 .../0/report.json | 668 ++++++ .../rla_esam-best-head-evaluation/0/DONE | 0 .../0/report.json | 213 ++ .../rla_esam-evaluation/0.toml | 51 + .../rla_esam-evaluation/0/DONE | 0 .../rla_esam-evaluation/0/report.json | 209 ++ .../rla_esam-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 217 ++ .../rla_mfb-best-head-evaluation/0/DONE | 0 .../0/report.json | 239 +++ .../rla_mfb-evaluation/0.toml | 54 + .../rla_mfb-evaluation/0/DONE | 0 .../rla_mfb-evaluation/0/report.json | 235 +++ .../rla_mfb-greedy-heads-evaluation/0/DONE | 0 .../0/report.json | 245 +++ .../0/DONE | 0 .../0/report.json | 203 ++ .../rla_rank2_noise0.0001-evaluation/0.toml | 41 + .../rla_rank2_noise0.0001-evaluation/0/DONE | 0 .../0/report.json | 199 ++ .../0/DONE | 0 .../0/report.json | 210 ++ .../0/DONE | 0 .../0/report.json | 203 ++ .../rla_rank2_noise0.001-evaluation/0.toml | 41 + .../rla_rank2_noise0.001-evaluation/0/DONE | 0 .../0/report.json | 199 ++ .../0/DONE | 0 .../0/report.json | 206 ++ .../0/DONE | 0 .../0/report.json | 203 ++ .../rla_rank2_noise1e-05-evaluation/0.toml | 41 + .../rla_rank2_noise1e-05-evaluation/0/DONE | 0 .../0/report.json | 199 ++ .../0/DONE | 0 .../0/report.json | 207 ++ .../0/DONE | 0 .../0/report.json | 203 ++ .../rla_rank4_noise0.0001-evaluation/0.toml | 41 + .../rla_rank4_noise0.0001-evaluation/0/DONE | 0 .../0/report.json | 199 ++ .../0/DONE | 0 .../0/report.json | 208 ++ .../0/DONE | 0 .../0/report.json | 203 ++ .../rla_rank4_noise0.001-evaluation/0.toml | 41 + .../rla_rank4_noise0.001-evaluation/0/DONE | 0 .../0/report.json | 199 ++ .../0/DONE | 0 .../0/report.json | 207 ++ .../0/DONE | 0 .../0/report.json | 203 ++ .../rla_rank4_noise1e-05-evaluation/0.toml | 41 + .../rla_rank4_noise1e-05-evaluation/0/DONE | 0 .../0/report.json | 199 ++ .../0/DONE | 0 .../0/report.json | 211 ++ .../0/DONE | 0 .../0/report.json | 203 ++ .../rla_rank8_noise0.0001-evaluation/0.toml | 41 + .../rla_rank8_noise0.0001-evaluation/0/DONE | 0 .../0/report.json | 199 ++ .../0/DONE | 0 .../0/report.json | 211 ++ .../0/DONE | 0 .../0/report.json | 203 ++ .../rla_rank8_noise0.001-evaluation/0.toml | 41 + .../rla_rank8_noise0.001-evaluation/0/DONE | 0 .../0/report.json | 199 ++ .../0/DONE | 0 .../0/report.json | 212 ++ .../0/DONE | 0 .../0/report.json | 203 ++ .../rla_rank8_noise1e-05-evaluation/0.toml | 41 + .../rla_rank8_noise1e-05-evaluation/0/DONE | 0 .../0/report.json | 199 ++ .../0/DONE | 0 .../0/report.json | 210 ++ .../paper/exp/selected_integrated_configs.csv | 31 + tabm_integrated/paper/lib/__init__.py | 5 + tabm_integrated/paper/lib/cf_fisd.py | 125 ++ tabm_integrated/paper/lib/data.py | 397 ++++ tabm_integrated/paper/lib/deep.py | 651 ++++++ tabm_integrated/paper/lib/env.py | 28 + tabm_integrated/paper/lib/metrics.py | 66 + tabm_integrated/paper/lib/util.py | 517 +++++ tabm_integrated/paper/pixi.toml | 58 + tabm_integrated/paper/pyproject.toml | 13 + .../paper/tools/cf_fisd_teachers.py | 310 +++ .../tools/aggregate_integrated_results.py | 368 +++- .../tools/generate_integrated_configs.py | 313 ++- .../tools/run_integrated_matrix.sh | 163 +- 2436 files changed, 278860 insertions(+), 25783 deletions(-) create mode 100644 tabm_integrated/paper/.gitignore create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/manifest_baseline_fidelity.txt create mode 100644 tabm_integrated/paper/exp/integrated/manifest_final.txt create mode 100644 tabm_integrated/paper/exp/integrated/manifest_smoke.txt create mode 100644 tabm_integrated/paper/exp/integrated/manifest_sweeps.txt create mode 100644 tabm_integrated/paper/exp/integrated/manifest_sweeps_remaining.txt delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1/report.json delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2.toml delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2/DONE delete mode 100644 tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/selected_integrated_configs.csv create mode 100644 tabm_integrated/paper/lib/__init__.py create mode 100644 tabm_integrated/paper/lib/cf_fisd.py create mode 100644 tabm_integrated/paper/lib/data.py create mode 100644 tabm_integrated/paper/lib/deep.py create mode 100644 tabm_integrated/paper/lib/env.py create mode 100644 tabm_integrated/paper/lib/metrics.py create mode 100644 tabm_integrated/paper/lib/util.py create mode 100644 tabm_integrated/paper/pixi.toml create mode 100644 tabm_integrated/paper/pyproject.toml create mode 100644 tabm_integrated/paper/tools/cf_fisd_teachers.py diff --git a/TEAM_MODULE_INVENTORY.md b/TEAM_MODULE_INVENTORY.md index bf769db56..3251e33a4 100644 --- a/TEAM_MODULE_INVENTORY.md +++ b/TEAM_MODULE_INVENTORY.md @@ -1,8 +1,54 @@ -# Team Module Inventory +# Team module inventory -- `RLA/`: Rank-low-rank adapter experiments and audited final evidence. Integrated flag: `model.rla_enabled`. -- `ESAM/`: Efficient/adaptive SAM training-loop variant. Integrated flag: `model.esam_enabled`. -- `MFB/`: member/feature bagging notebook implementation. Integrated flag: `model.mfb_enabled`. -- `cf_fisd_recovered/`: CF-FISD feature-importance diagnostics and model hooks. Integrated flag: `model.cf_fisd_enabled`. +## RLA +- Source files: `RLA/paper/lib/deep.py`, `RLA/paper/bin/model.py`, `RLA/paper/tests/test_rla.py`. +- Integrated destination files: `tabm_integrated/paper/lib/deep.py`, `tabm_integrated/paper/bin/run_integrated.py`. +- Configs: official baseline configs plus `model.rla_rank`, `model.rla_first_only`, `model.rla_additive`, `model.rla_init`, `model.rla_base_preserve_noise`, `rla_adapter_lr_multiplier`, `rla_extra_paths_freeze_fraction`. +- Scripts: `tabm_integrated/tools/generate_integrated_configs.py`, `tabm_integrated/tools/run_integrated_matrix.sh`, `tabm_integrated/tools/aggregate_integrated_results.py`. +- Notebooks: none required for RLA. +- Result artifacts: `tabm_integrated/paper/exp/integrated/**/rla*`, `tabm_integrated/paper/exp/final_integrated_summary.csv`, `tabm_integrated/paper/exp/final_integrated_audit.csv`. +- Reproduction command: `N_GPUS=16 tabm_integrated/tools/run_integrated_matrix.sh exp/integrated/manifest_sweeps.txt` followed by final selection and final matrix commands. +- Assumptions: RLA rank-r adapters are used only in TabM-style BatchEnsemble layers. +- Known limitations: CF-FISD alignment with RLA combined variants regularizes the first RLA input path (`R[:, :, 0]`). -The integrated runner keeps the official TabM PLR baseline when all four flags are off and exposes module-only plus combined variants through `tabm_integrated/tools/generate_integrated_configs.py`. +## ESAM +- Source files: `ESAM/code/bin/model.py`, `ESAM/code/lib/*`. +- Integrated destination files: `tabm_integrated/paper/bin/run_integrated.py`, `tabm_integrated/paper/lib/*`. +- Configs: `use_esam`, `esam_rho`, `esam_eps`, `esam_adapter_only`, `esam_memberwise`, warmup/start/end epoch flags, diagnostics flags. +- Scripts: same integrated generator, queue launcher, and aggregator. +- Notebooks: none required for ESAM. +- Result artifacts: `tabm_integrated/paper/exp/integrated/**/esam*` and final CSV/report artifacts. +- Reproduction command: generate configs, then run the smoke/sweep/final manifests with `N_GPUS=16`. +- Assumptions: adapter-only ESAM perturbs member-specific TabM adapters, including RLA `R/S` when present. +- Known limitations: diagnostics are disabled by default to keep committed artifacts compact. + +## MFB +- Source files: `MFB/code/ramp_ncl_mfb_tc5_end_to_end.ipynb`. +- Integrated destination files: `tabm_integrated/paper/bin/run_integrated.py`. +- Configs: `model.mfb.enabled`, `mask_mode`, `mask_granularity`, `keep_rate`, `inverted_scaling`, `use_soft_mask`, `mask_strength`, `anchor_fraction`, `warmup_epochs`, `mask_seed`. +- Scripts: same integrated generator, queue launcher, and aggregator. +- Notebooks: source notebook above; integrated runner ports the legacy member-fixed feature-group MFB path used by the requested keep-rate sweep. +- Result artifacts: `tabm_integrated/paper/exp/integrated/**/mfb*` and final CSV/report artifacts. +- Reproduction command: run the smoke/sweep/final manifests with `N_GPUS=16`. +- Assumptions: requested MFB sweep uses the notebook's legacy hard member-fixed feature-group mask, not the RAMP router ablations. +- Known limitations: RAMP router/fidelity variants from the notebook are not part of the requested keep-rate sweep. + +## CF-FISD +- Source files: `cf_fisd_recovered/paper/lib/cf_fisd.py`, `cf_fisd_recovered/paper/bin/cf_fisd_teachers.py`, `cf_fisd_recovered/paper/tools/generate_cf_fisd_configs.py`, teacher importance arrays under `cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/`. +- Integrated destination files: `tabm_integrated/paper/lib/cf_fisd.py`, `tabm_integrated/paper/tools/cf_fisd_teachers.py`, `tabm_integrated/paper/bin/run_integrated.py`. +- Configs: `cf_fisd.lambda`, `cf_fisd.variant`, `cf_fisd.dataset_name`, `cf_fisd.teacher_dir`, `cf_fisd.teacher_names`, optional member groups. +- Scripts: same integrated generator, queue launcher, and aggregator. +- Notebooks: none required for CF-FISD. +- Result artifacts: `tabm_integrated/paper/exp/integrated/**/cf_fisd*` and final CSV/report artifacts. +- Reproduction command: run generated manifests after verifying teacher arrays exist. +- Assumptions: bundled teammate teacher importance arrays are reused for the five TabReD datasets. +- Known limitations: if teacher arrays are regenerated, final configs should point to the regenerated teacher directory. + +## End-to-end reproduction +1. `cd tabm_integrated/paper && /workspace/.venvs/tabm_integrated/bin/python ../tools/generate_integrated_configs.py --stage initial --clean` +2. `N_GPUS=16 tabm_integrated/tools/run_integrated_matrix.sh exp/integrated/manifest_baseline_fidelity.txt` +3. `N_GPUS=16 tabm_integrated/tools/run_integrated_matrix.sh exp/integrated/manifest_smoke.txt` +4. `N_GPUS=16 tabm_integrated/tools/run_integrated_matrix.sh exp/integrated/manifest_sweeps.txt` +5. `/workspace/.venvs/tabm_integrated/bin/python tabm_integrated/tools/aggregate_integrated_results.py --stage select-final` +6. `N_GPUS=16 tabm_integrated/tools/run_integrated_matrix.sh exp/integrated/manifest_final.txt` +7. `/workspace/.venvs/tabm_integrated/bin/python tabm_integrated/tools/aggregate_integrated_results.py --stage final` diff --git a/tabm_integrated/FINAL_EXPERIMENT_REPORT.md b/tabm_integrated/FINAL_EXPERIMENT_REPORT.md index 4a12a940b..ad80ccbb3 100644 --- a/tabm_integrated/FINAL_EXPERIMENT_REPORT.md +++ b/tabm_integrated/FINAL_EXPERIMENT_REPORT.md @@ -1,56 +1,81 @@ -# Final Integrated TabM Experiment Report +# Integrated TabM final experiment report -Mean ± std over available seeds. Safe claims require all 3 seeds. +Official TabM PLR/PiecewiseLinearEmbeddings baseline configs are preserved exactly; variants add only module flags for RLA, ESAM, MFB, and CF-FISD. -| dataset | variant | metric | direction | inference | precision | mean ± std | delta | percent delta | n | status | config path | result path | -|---|---|---|---|---|---|---:|---:|---:|---:|---|---|---| -| sberbank-housing | baseline_plr | rmse | lower | mean | bfloat16 | 11.5057 ± 0.0432 | 0 | 0.000% | 3 | baseline | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation | -| sberbank-housing | rla_only | rmse | lower | mean | bfloat16 | 11.5041 ± 0.00471 | 0.00157171 | 0.014% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation | -| sberbank-housing | esam_only | rmse | lower | mean | bfloat16 | 11.5065 ± 0.0432 | -0.000818784 | -0.007% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation | -| sberbank-housing | mfb_only | rmse | lower | mean | bfloat16 | 11.5257 ± 0.0111 | -0.0199767 | -0.174% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation | -| sberbank-housing | cf_fisd_only | rmse | lower | mean | bfloat16 | 11.5355 ± 0.0054 | -0.0297966 | -0.259% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation | -| sberbank-housing | all_four_combined | rmse | lower | mean | bfloat16 | 11.5358 ± 0.00515 | -0.030061 | -0.261% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation | -| sberbank-housing | all_minus_rla | rmse | lower | mean | bfloat16 | 11.5352 ± 0.00412 | -0.0294744 | -0.256% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation | -| sberbank-housing | all_minus_esam | rmse | lower | mean | bfloat16 | 11.5344 ± 0.00432 | -0.028726 | -0.250% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation | -| sberbank-housing | all_minus_mfb | rmse | lower | mean | bfloat16 | 11.5315 ± 0.00449 | -0.0257712 | -0.224% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation | -| sberbank-housing | all_minus_cf_fisd | rmse | lower | mean | bfloat16 | 11.5117 ± 0.0154 | -0.00600246 | -0.052% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation | -| ecom-offers | baseline_plr | roc-auc | higher | mean | bfloat16 | 0.576461 ± 0.000301 | 0 | 0.000% | 3 | baseline | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation | -| ecom-offers | rla_only | roc-auc | higher | mean | bfloat16 | 0.576807 ± 0.000199 | 0.000345614 | 0.060% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation | -| ecom-offers | esam_only | roc-auc | higher | mean | bfloat16 | 0.5766 ± 0.00017 | 0.000138612 | 0.024% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation | -| ecom-offers | mfb_only | roc-auc | higher | mean | bfloat16 | 0.577706 ± 0.0034 | 0.00124474 | 0.216% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation | -| ecom-offers | cf_fisd_only | roc-auc | higher | mean | bfloat16 | 0.573314 ± 0.000392 | -0.00314765 | -0.546% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation | -| ecom-offers | all_four_combined | roc-auc | higher | mean | bfloat16 | 0.575357 ± 0.00178 | -0.00110457 | -0.192% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation | -| ecom-offers | all_minus_rla | roc-auc | higher | mean | bfloat16 | 0.572037 ± 0.00203 | -0.0044245 | -0.768% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation | -| ecom-offers | all_minus_esam | roc-auc | higher | mean | bfloat16 | 0.572005 ± 0.00109 | -0.00445646 | -0.773% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation | -| ecom-offers | all_minus_mfb | roc-auc | higher | mean | bfloat16 | 0.574369 ± 0.00047 | -0.00209235 | -0.363% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation | -| ecom-offers | all_minus_cf_fisd | roc-auc | higher | mean | bfloat16 | 0.576756 ± 0.00348 | 0.000294538 | 0.051% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation | -| homesite-insurance | baseline_plr | roc-auc | higher | mean | bfloat16 | 0.953231 ± 0.0006 | 0 | 0.000% | 3 | baseline | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation | -| homesite-insurance | rla_only | roc-auc | higher | mean | bfloat16 | 0.951962 ± 0.000732 | -0.00126848 | -0.133% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation | -| homesite-insurance | esam_only | roc-auc | higher | mean | bfloat16 | 0.953275 ± 0.00054 | 4.45755e-05 | 0.005% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation | -| homesite-insurance | mfb_only | roc-auc | higher | mean | bfloat16 | 0.950035 ± 0.000335 | -0.00319556 | -0.335% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation | -| homesite-insurance | cf_fisd_only | roc-auc | higher | mean | bfloat16 | 0.954365 ± 0.000453 | 0.001134 | 0.119% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation | -| homesite-insurance | all_four_combined | roc-auc | higher | mean | bfloat16 | 0.949588 ± 0.000717 | -0.00364296 | -0.382% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation | -| homesite-insurance | all_minus_rla | roc-auc | higher | mean | bfloat16 | 0.95047 ± 0.000475 | -0.00276044 | -0.290% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation | -| homesite-insurance | all_minus_esam | roc-auc | higher | mean | bfloat16 | 0.949443 ± 0.00153 | -0.0037874 | -0.397% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation | -| homesite-insurance | all_minus_mfb | roc-auc | higher | mean | bfloat16 | 0.95332 ± 0.000513 | 8.89334e-05 | 0.009% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation | -| homesite-insurance | all_minus_cf_fisd | roc-auc | higher | mean | bfloat16 | 0.94591 ± 0.00176 | -0.00732119 | -0.768% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation | -| cooking-time | baseline_plr | rmse | lower | mean | bfloat16 | 2.56457 ± 0.0029 | 0 | 0.000% | 3 | baseline | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation | -| cooking-time | rla_only | rmse | lower | mean | bfloat16 | 2.56088 ± 0.0138 | 0.00368773 | 0.144% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation | -| cooking-time | esam_only | rmse | lower | mean | bfloat16 | 2.56386 ± 0.00279 | 0.000706405 | 0.028% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation | -| cooking-time | mfb_only | rmse | lower | mean | bfloat16 | 2.5445 ± 0.00955 | 0.0200701 | 0.783% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation | -| cooking-time | cf_fisd_only | rmse | lower | mean | bfloat16 | 2.56324 ± 0.00279 | 0.00133126 | 0.052% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation | -| cooking-time | all_four_combined | rmse | lower | mean | bfloat16 | 2.54421 ± 0.00684 | 0.0203545 | 0.794% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation | -| cooking-time | all_minus_rla | rmse | lower | mean | bfloat16 | 2.54693 ± 0.00325 | 0.0176392 | 0.688% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation | -| cooking-time | all_minus_esam | rmse | lower | mean | bfloat16 | 2.54322 ± 0.00473 | 0.0213521 | 0.833% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation | -| cooking-time | all_minus_mfb | rmse | lower | mean | bfloat16 | 2.55993 ± 0.0125 | 0.00464007 | 0.181% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation | -| cooking-time | all_minus_cf_fisd | rmse | lower | mean | bfloat16 | 2.54518 ± 0.0029 | 0.0193905 | 0.756% | 3 | win | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation | -| delivery-eta | baseline_plr | rmse | lower | mean | bfloat16 | 1.85423 ± 0.0129 | 0 | 0.000% | 3 | baseline | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation | -| delivery-eta | rla_only | rmse | lower | mean | bfloat16 | 1.87913 ± 0.00475 | -0.0249039 | -1.343% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation | -| delivery-eta | esam_only | rmse | lower | mean | bfloat16 | 1.85645 ± 0.0134 | -0.00222041 | -0.120% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation | -| delivery-eta | mfb_only | rmse | lower | mean | bfloat16 | 1.88817 ± 0.0102 | -0.0339417 | -1.831% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation | -| delivery-eta | cf_fisd_only | rmse | lower | mean | bfloat16 | 1.85934 ± 0.0133 | -0.00511517 | -0.276% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation | -| delivery-eta | all_four_combined | rmse | lower | mean | bfloat16 | 1.8809 ± 0.0154 | -0.0266763 | -1.439% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation | -| delivery-eta | all_minus_rla | rmse | lower | mean | bfloat16 | 1.87905 ± 0.0243 | -0.0248174 | -1.338% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation | -| delivery-eta | all_minus_esam | rmse | lower | mean | bfloat16 | 1.87891 ± 0.0186 | -0.0246832 | -1.331% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation | -| delivery-eta | all_minus_mfb | rmse | lower | mean | bfloat16 | 1.86875 ± 0.00678 | -0.0145174 | -0.783% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation | -| delivery-eta | all_minus_cf_fisd | rmse | lower | mean | bfloat16 | 1.87292 ± 0.0162 | -0.0186869 | -1.008% | 3 | loss | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0.toml | /workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation | +## Final 3-seed results + +| dataset | variant | source config | metric | inference | mean ± std | delta | status | +|---|---|---|---|---|---:|---:|---| +| sberbank-housing | baseline_plr | baseline_plr | RMSE | mean | 0.234418 ± 0.00354822 | 0 | baseline | +| sberbank-housing | best_rla_only | rla_rank2_noise0.001 | RMSE | greedy-heads | 0.232368 ± 0.00228551 | 0.0020499 | weak_win | +| sberbank-housing | best_esam_only | esam_rho0.0025 | RMSE | greedy-heads | 0.234406 ± 0.00401172 | 1.17846e-05 | weak_win | +| sberbank-housing | best_mfb_only | mfb_keep0.8 | RMSE | greedy-heads | 0.2351 ± 0.00395661 | -0.000681772 | loss | +| sberbank-housing | best_cf_fisd_only | cf_fisd_only | RMSE | greedy-heads | 0.233093 ± 0.000585082 | 0.0013248 | weak_win | +| sberbank-housing | best_combined | mfb_cf_fisd | RMSE | greedy-heads | 0.234979 ± 0.00337063 | -0.000560913 | loss | +| ecom-offers | baseline_plr | baseline_plr | AUROC | mean | 0.590058 ± 0.000970364 | 0 | baseline | +| ecom-offers | best_rla_only | rla_rank2_noise0.0001 | AUROC | best-head | 0.598447 ± 0.00279735 | 0.00838888 | clear_win | +| ecom-offers | best_esam_only | esam_rho0.001 | AUROC | greedy-heads | 0.60042 ± 0.00349335 | 0.0103616 | clear_win | +| ecom-offers | best_mfb_only | mfb_keep0.7 | AUROC | best-head | 0.609516 ± 0.00208948 | 0.0194584 | clear_win | +| ecom-offers | best_cf_fisd_only | cf_fisd_lambda0.05 | AUROC | greedy-heads | 0.600205 ± 0.00345825 | 0.0101468 | clear_win | +| ecom-offers | best_combined | mfb_cf_fisd | AUROC | greedy-heads | 0.605211 ± 0.00322248 | 0.0151529 | clear_win | +| homesite-insurance | baseline_plr | baseline_plr | AUROC | mean | 0.962494 ± 0.000178079 | 0 | baseline | +| homesite-insurance | best_rla_only | rla_rank4_noise1e-05 | AUROC | greedy-heads | 0.962564 ± 0.000608529 | 6.98613e-05 | weak_win | +| homesite-insurance | best_esam_only | esam_only | AUROC | greedy-heads | 0.962732 ± 0.00044923 | 0.000237685 | clear_win | +| homesite-insurance | best_mfb_only | mfb_keep0.95 | AUROC | greedy-heads | 0.962382 ± 0.00045772 | -0.000111855 | loss | +| homesite-insurance | best_cf_fisd_only | cf_fisd_only | AUROC | greedy-heads | 0.962908 ± 0.000474027 | 0.000413458 | clear_win | +| homesite-insurance | best_combined | mfb_cf_fisd | AUROC | greedy-heads | 0.962849 ± 0.000437968 | 0.000354836 | clear_win | +| cooking-time | baseline_plr | baseline_plr | RMSE | mean | 0.480381 ± 0.000123385 | 0 | baseline | +| cooking-time | best_rla_only | rla_rank4_noise0.001 | RMSE | greedy-heads | 0.480146 ± 0.00016089 | 0.000234391 | clear_win | +| cooking-time | best_esam_only | esam_rho0.0025 | RMSE | greedy-heads | 0.480065 ± 8.02121e-05 | 0.000315778 | clear_win | +| cooking-time | best_mfb_only | mfb_keep0.8 | RMSE | greedy-heads | 0.479973 ± 0.000194552 | 0.000407806 | clear_win | +| cooking-time | best_cf_fisd_only | cf_fisd_lambda0.05 | RMSE | greedy-heads | 0.480337 ± 4.64953e-05 | 4.32134e-05 | weak_win | +| cooking-time | best_combined | rla_esam | RMSE | greedy-heads | 0.480026 ± 9.6554e-05 | 0.000354592 | clear_win | +| delivery-eta | baseline_plr | baseline_plr | RMSE | mean | 0.550226 ± 0.000582292 | 0 | baseline | +| delivery-eta | best_rla_only | rla_rank2_noise0.001 | RMSE | greedy-heads | 0.551407 ± 0.00109716 | -0.00118071 | loss | +| delivery-eta | best_esam_only | esam_only | RMSE | greedy-heads | 0.551148 ± 0.000504719 | -0.000922076 | loss | +| delivery-eta | best_mfb_only | mfb_keep0.7 | RMSE | greedy-heads | 0.551366 ± 0.00171387 | -0.00114032 | loss | +| delivery-eta | best_cf_fisd_only | cf_fisd_lambda0.05 | RMSE | greedy-heads | 0.552434 ± 0.00139117 | -0.00220824 | loss | +| delivery-eta | best_combined | rla_esam | RMSE | greedy-heads | 0.552336 ± 0.000198974 | -0.00211043 | loss | + +## Validation-selected configs + +| dataset | final variant | selected sweep variant | inference | validation metric | +|---|---|---|---|---:| +| sberbank-housing | baseline_plr | baseline_plr | mean | | +| sberbank-housing | best_rla_only | rla_rank2_noise0.001 | greedy-heads | 0.2328613038082034 | +| sberbank-housing | best_esam_only | esam_rho0.0025 | greedy-heads | 0.23295802298295287 | +| sberbank-housing | best_mfb_only | mfb_keep0.8 | greedy-heads | 0.23237596587359466 | +| sberbank-housing | best_cf_fisd_only | cf_fisd_only | greedy-heads | 0.23327114512588593 | +| sberbank-housing | best_combined | mfb_cf_fisd | greedy-heads | 0.23320900639668657 | +| ecom-offers | baseline_plr | baseline_plr | mean | | +| ecom-offers | best_rla_only | rla_rank2_noise0.0001 | best-head | 0.6396541901648656 | +| ecom-offers | best_esam_only | esam_rho0.001 | greedy-heads | 0.641845068859757 | +| ecom-offers | best_mfb_only | mfb_keep0.7 | best-head | 0.6547032702847417 | +| ecom-offers | best_cf_fisd_only | cf_fisd_lambda0.05 | greedy-heads | 0.6419149054445072 | +| ecom-offers | best_combined | mfb_cf_fisd | greedy-heads | 0.6506783465179156 | +| homesite-insurance | baseline_plr | baseline_plr | mean | | +| homesite-insurance | best_rla_only | rla_rank4_noise1e-05 | greedy-heads | 0.9591329763748824 | +| homesite-insurance | best_esam_only | esam_only | greedy-heads | 0.958991133281215 | +| homesite-insurance | best_mfb_only | mfb_keep0.95 | greedy-heads | 0.9592957095194116 | +| homesite-insurance | best_cf_fisd_only | cf_fisd_only | greedy-heads | 0.9594844321379927 | +| homesite-insurance | best_combined | mfb_cf_fisd | greedy-heads | 0.9597570226838343 | +| cooking-time | baseline_plr | baseline_plr | mean | | +| cooking-time | best_rla_only | rla_rank4_noise0.001 | greedy-heads | 0.4624107384246987 | +| cooking-time | best_esam_only | esam_rho0.0025 | greedy-heads | 0.46246732196836127 | +| cooking-time | best_mfb_only | mfb_keep0.8 | greedy-heads | 0.4624822561623469 | +| cooking-time | best_cf_fisd_only | cf_fisd_lambda0.05 | greedy-heads | 0.4628482268025946 | +| cooking-time | best_combined | rla_esam | greedy-heads | 0.4624234187671886 | +| delivery-eta | baseline_plr | baseline_plr | mean | | +| delivery-eta | best_rla_only | rla_rank2_noise0.001 | greedy-heads | 0.5552720495549479 | +| delivery-eta | best_esam_only | esam_only | greedy-heads | 0.5551513290724005 | +| delivery-eta | best_mfb_only | mfb_keep0.7 | greedy-heads | 0.5551008108444078 | +| delivery-eta | best_cf_fisd_only | cf_fisd_lambda0.05 | greedy-heads | 0.5556691040453957 | +| delivery-eta | best_combined | rla_esam | greedy-heads | 0.5552988041877751 | + +## Module wins vs baseline + +- `best_rla_only`: sberbank-housing, ecom-offers, homesite-insurance, cooking-time +- `best_esam_only`: sberbank-housing, ecom-offers, homesite-insurance, cooking-time +- `best_mfb_only`: ecom-offers, cooking-time +- `best_cf_fisd_only`: sberbank-housing, ecom-offers, homesite-insurance, cooking-time +- `best_combined`: ecom-offers, homesite-insurance, cooking-time diff --git a/tabm_integrated/paper/.gitignore b/tabm_integrated/paper/.gitignore new file mode 100644 index 000000000..48d800d9e --- /dev/null +++ b/tabm_integrated/paper/.gitignore @@ -0,0 +1,5 @@ +cache/ +__pycache__/ +.pytest_cache/ +*.pyc +cache diff --git a/tabm_integrated/paper/bin/run_integrated.py b/tabm_integrated/paper/bin/run_integrated.py index 448d6961d..6dabe48d5 100644 --- a/tabm_integrated/paper/bin/run_integrated.py +++ b/tabm_integrated/paper/bin/run_integrated.py @@ -1,198 +1,1686 @@ -from __future__ import annotations -import argparse, json, math, random, time, tomllib +import math +import json +import shutil +import statistics +import sys from pathlib import Path -from typing import Any +from typing import Any, Literal + +import delu import numpy as np -import torch -from sklearn.metrics import mean_squared_error, roc_auc_score -from torch import nn -from torch.utils.data import DataLoader, TensorDataset -import tabm import rtdl_num_embeddings +import scipy +import torch +import torch.nn as nn +import torch.utils.tensorboard +from loguru import logger +from torch import Tensor +from tqdm import tqdm +from typing_extensions import NotRequired, TypedDict + +if __name__ == '__main__': + _cwd = Path.cwd() + assert _cwd.joinpath( + 'pixi.toml' + ).exists(), 'The script must be run from the `paper/` directory' + sys.path.append(str(_cwd)) + del _cwd + +import lib +import lib.data +import lib.deep +import lib.env +import lib.cf_fisd +from lib import KWArgs, PartKey + + +def _get_first_ensemble_layer( + backbone: lib.deep.MLP, +) -> lib.deep.LinearEfficientEnsemble: + if isinstance(backbone, lib.deep.MLP): + return backbone.blocks[0][0] # type: ignore[code] + else: + raise RuntimeError(f'Unsupported backbone: {backbone}') + + +@torch.inference_mode() +def _init_first_adapter( + weight: Tensor, + distribution: Literal['normal', 'random-signs'], + init_sections: list[int], +) -> None: + """Initialize the first adapter. + + NOTE + The `init_sections` argument is a historical artifact that accidentally leaked + from irrelevant experiments to the final models. Perhaps, the code related + to `init_sections` can be simply removed, but this was not tested. + """ + assert weight.ndim == 2 + assert weight.shape[1] == sum(init_sections) + + if distribution == 'normal': + init_fn_ = nn.init.normal_ + elif distribution == 'random-signs': + init_fn_ = lib.deep.init_random_signs_ + else: + raise ValueError(f'Unknown distribution: {distribution}') + + section_bounds = [0, *torch.tensor(init_sections).cumsum(0).tolist()] + for i in range(len(init_sections)): + # NOTE + # As noted above, this section-based initialization is an arbitrary historical + # artifact. Consider the first adapter of one ensemble member. + # This adapter vector is implicitly split into "sections", + # where one section corresponds to one feature. The code below ensures that + # the adapter weights in one section are initialized with the same random value + # from the given distribution. + w = torch.empty((len(weight), 1), dtype=weight.dtype, device=weight.device) + init_fn_(w) + weight[:, section_bounds[i] : section_bounds[i + 1]] = w + + +def _get_first_rankr_layer( + backbone: lib.deep.MLP, +) -> lib.deep.LinearEfficientEnsembleRankR: + if isinstance(backbone, lib.deep.MLP): + return backbone.blocks[0][0] # type: ignore[code] + raise RuntimeError(f'Unsupported backbone: {backbone}') + + +@torch.inference_mode() +def _init_first_adapter_rankr( + R: Tensor, + distribution: Literal['normal', 'random-signs'], + init_sections: list[int], +) -> None: + """Initialise the first-layer R parameter (k, d_in, rank). + + Each rank column receives an independent section-wise random sign + (or normal) draw, then the entire R tensor is scaled by 1/sqrt(rank) + so the sum over the rank axis preserves the variance of the rank-1 + baseline at initialisation. + """ + assert R.ndim == 3 + k, d_in, rank = R.shape + assert d_in == sum(init_sections) + + if distribution == 'normal': + init_fn_ = nn.init.normal_ + elif distribution == 'random-signs': + init_fn_ = lib.deep.init_random_signs_ + else: + raise ValueError(f'Unknown distribution: {distribution}') + + section_bounds = [0, *torch.tensor(init_sections).cumsum(0).tolist()] + for j in range(rank): + for i in range(len(init_sections)): + w = torch.empty((k, 1), dtype=R.dtype, device=R.device) + init_fn_(w) + R[:, section_bounds[i] : section_bounds[i + 1], j] = w + R.mul_(rank**-0.5) + + +def _replace_first_with_rankr( + backbone: lib.deep.MLP, + *, + k: int, + rank: int, + additive: bool, + init_mode: str = 'variance_preserving', + base_preserve_noise: float = 1e-3, +) -> None: + """Replace the first ensemble linear layer with the rank-r variant. + + The remaining layers stay as the baseline rank-1 class. This implements + the RLA-first variant from Section 3.1 of the spec. + + NOTE on RNG isolation: ``LinearEfficientEnsembleRankR.__init__`` calls + ``reset_parameters`` which consumes global RNG for its (about-to-be + overwritten) weight and bias initialisation. Without isolation, that + extra draw drifts the RNG state and ``_init_first_adapter_basepreserve`` + below would then sample a different sequence of random signs than the + baseline construction did, breaking full-model bit-equivalence at + init. Save+restore the global RNG state around the construction. + """ + first = _get_first_ensemble_layer(backbone) + rng_state = torch.random.get_rng_state() + try: + new = lib.deep.LinearEfficientEnsembleRankR( + in_features=first.in_features, + out_features=first.out_features, + bias=first.bias is not None, + k=k, + rank=rank, + scaling_init='ones', + additive=additive, + init_mode=init_mode, + base_preserve_noise=base_preserve_noise, + ) + finally: + torch.random.set_rng_state(rng_state) + # Re-use the same backbone weights for stability of the comparison. + with torch.inference_mode(): + new.weight.copy_(first.weight) + if first.bias is not None and new.bias is not None: + new.bias.copy_(first.bias) + # Splice into the backbone (paper/lib MLP block 0 sequential[0]). + backbone.blocks[0][0] = new # type: ignore[code] + + +@torch.inference_mode() +def _init_first_adapter_basepreserve( + R: Tensor, + distribution: Literal['normal', 'random-signs'], + init_sections: list[int], +) -> None: + """Re-initialise *only path 0* of a rank-r R parameter using the + section-based scheme. Extra paths (j >= 1) are left untouched, + preserving the base-preserving zero/tiny-noise init. + """ + assert R.ndim == 3 + k, d_in, rank = R.shape + assert d_in == sum(init_sections) + + if distribution == 'normal': + init_fn_ = nn.init.normal_ + elif distribution == 'random-signs': + init_fn_ = lib.deep.init_random_signs_ + else: + raise ValueError(f'Unknown distribution: {distribution}') + + section_bounds = [0, *torch.tensor(init_sections).cumsum(0).tolist()] + for i in range(len(init_sections)): + w = torch.empty((k, 1), dtype=R.dtype, device=R.device) + init_fn_(w) + R[:, section_bounds[i] : section_bounds[i + 1], 0] = w + # Note: do NOT scale by 1/sqrt(r) here — only path 0 contributes at init, + # so it must carry the full baseline magnitude. + + + + +def _mean_pairwise_jaccard(mask: np.ndarray) -> float: + if mask.shape[0] < 2: + return 1.0 + values: list[float] = [] + for i in range(mask.shape[0]): + for j in range(i + 1, mask.shape[0]): + union = np.logical_or(mask[i], mask[j]).sum() + values.append(1.0 if union == 0 else float(np.logical_and(mask[i], mask[j]).sum() / union)) + return float(np.mean(values)) if values else 1.0 + + +def _mfb_mask_coverage_stats(mask: np.ndarray) -> dict[str, float]: + features_per_member = mask.sum(axis=1) + members_per_feature = mask.sum(axis=0) + return { + 'keep_rate_actual': float(mask.mean()), + 'min_features_per_member': float(features_per_member.min()), + 'mean_features_per_member': float(features_per_member.mean()), + 'max_features_per_member': float(features_per_member.max()), + 'min_members_per_feature': float(members_per_feature.min()), + 'mean_members_per_feature': float(members_per_feature.mean()), + 'max_members_per_feature': float(members_per_feature.max()), + 'mean_pairwise_jaccard': _mean_pairwise_jaccard(mask), + } + + +def _make_mfb_feature_group_mask( + *, + k: int, + feature_widths: list[int], + keep_rate: float, + seed: int, + anchor_fraction: float = 0.0, + protected_feature_ids: None | list[int] = None, + ensure_each_feature_seen: bool = True, + ensure_each_member_nonempty: bool = True, +) -> tuple[np.ndarray, np.ndarray, dict[str, float]]: + feature_index: list[int] = [] + for feature_id, width in enumerate(feature_widths): + feature_index.extend([feature_id] * int(width)) + n_features = len(feature_widths) + gen = np.random.RandomState(seed) + feature_mask = (gen.rand(k, n_features) < keep_rate).astype(np.float32) + protected = np.array([] if protected_feature_ids is None else protected_feature_ids, dtype=np.int64) + n_anchor = max(0, min(k, int(round(k * anchor_fraction)))) + if protected.size > 0: + feature_mask[:, protected] = 1.0 + if n_anchor > 0: + feature_mask[:n_anchor, :] = 1.0 + if ensure_each_member_nonempty: + for row in range(k): + if feature_mask[row].sum() == 0: + feature_mask[row, gen.randint(0, n_features)] = 1.0 + if ensure_each_feature_seen: + for col in range(n_features): + if feature_mask[:, col].sum() == 0: + feature_mask[gen.randint(0, k), col] = 1.0 + dim_mask = feature_mask[:, np.asarray(feature_index, dtype=np.int64)] + stats = _mfb_mask_coverage_stats(feature_mask) + stats.update({ + 'anchor_fraction_configured': float(anchor_fraction), + 'n_anchor_members': float(n_anchor), + 'core_fraction_configured': float(len(protected) / max(1, n_features)), + 'n_core_features': float(len(protected)), + }) + return feature_mask.astype(np.float32), dim_mask.astype(np.float32), stats + + +def _get_first_adapter_for_cf_fisd(backbone: lib.deep.MLP) -> Tensor: + first = backbone.blocks[0][0] # type: ignore[code] + if isinstance(first, lib.deep.LinearEfficientEnsembleRankR): + return first.R[:, :, 0] + if isinstance(first, lib.deep.LinearEfficientEnsemble): + if first.r is None: + raise RuntimeError('CF-FISD requires first-layer input scaling.') + return first.r + raise RuntimeError(f'Unsupported first ensemble layer for CF-FISD: {type(first)!r}') + +DEFAULT_SHARE_TRAINING_BATCHES = True + + +class Model(nn.Module): + """MLP & TabM.""" + + def __init__( + self, + *, + n_num_features: int, + cat_cardinalities: list[int], + n_classes: None | int, + backbone: dict, + bins: None | list[Tensor], # For piecewise-linear encoding/embeddings. + num_embeddings: None | dict = None, + arch_type: Literal[ + # Plain feed-forward network without any kind of ensembling. + 'plain', + # + # TabM + 'tabm', + # + # TabM-mini + 'tabm-mini', + # + # TabM-packed + 'tabm-packed', + # + # TabM. The first adapter is initialized from the normal distribution. + # This variant was not used in the paper, but it may be useful in practice. + 'tabm-normal', + # + # TabM-mini. The adapter is initialized from the normal distribution. + # This variant was not used in the paper. + 'tabm-mini-normal', + ], + k: None | int = None, + share_training_batches: bool = DEFAULT_SHARE_TRAINING_BATCHES, + rla_rank: int = 1, + rla_first_only: bool = False, + rla_additive: bool = False, + rla_init: Literal['variance_preserving', 'base_preserving'] = 'variance_preserving', + rla_base_preserve_noise: float = 1e-3, + mfb: None | dict[str, Any] = None, + ) -> None: + # >>> Validate arguments. + assert n_num_features >= 0 + assert n_num_features or cat_cardinalities + if arch_type == 'plain': + assert k is None + assert ( + share_training_batches + ), 'If `arch_type` is set to "plain", then `simple` must remain True' + else: + assert k is not None + assert k > 0 + + super().__init__() + + # >>> Continuous (numerical) features + first_adapter_sections = [] # See the comment in `_init_first_adapter`. + + if n_num_features == 0: + assert bins is None + self.num_module = None + d_num = 0 -LOWER = {'rmse'} - -def load_toml(path: Path) -> dict[str, Any]: - with path.open('rb') as f: - return tomllib.load(f) - -def dump_json(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2, sort_keys=True) + '\n') - -def set_seed(seed: int) -> None: - random.seed(seed); np.random.seed(seed); torch.manual_seed(seed); torch.cuda.manual_seed_all(seed) - torch.backends.cudnn.benchmark = True - -def read_arrays(data_path: Path): - info = json.loads((data_path / 'info.json').read_text()) - x_num, x_cat, y = {}, {}, {} - for part in ['train', 'val', 'test']: - nums = [] - for name in ['X_num', 'X_bin']: - p = data_path / f'{name}_{part}.npy' - if p.exists(): - nums.append(np.load(p, allow_pickle=True).astype('float32')) - y[part] = np.load(data_path / f'Y_{part}.npy', allow_pickle=True) - x_num[part] = np.concatenate(nums, axis=1).astype('float32') if nums else np.zeros((len(y[part]), 0), dtype='float32') - pcat = data_path / f'X_cat_{part}.npy' - if pcat.exists(): - x_cat[part] = np.load(pcat, allow_pickle=True).astype('int64') - return x_num, (x_cat if x_cat else None), y, info - -def standardize_num(x): - if x['train'].shape[1] == 0: + elif num_embeddings is None: + assert bins is None + self.num_module = None + d_num = n_num_features + first_adapter_sections.extend(1 for _ in range(n_num_features)) + + else: + if bins is None: + self.num_module = lib.deep.make_module( + **num_embeddings, n_features=n_num_features + ) + else: + assert num_embeddings['type'].startswith('PiecewiseLinearEmbeddings') + self.num_module = lib.deep.make_module(**num_embeddings, bins=bins) + d_num = n_num_features * num_embeddings['d_embedding'] + first_adapter_sections.extend( + num_embeddings['d_embedding'] for _ in range(n_num_features) + ) + + # >>> Categorical features + self.cat_module = ( + lib.deep.OneHotEncoding0d(cat_cardinalities) if cat_cardinalities else None + ) + first_adapter_sections.extend(cat_cardinalities) + d_cat = sum(cat_cardinalities) + + # >>> Backbone + d_flat = d_num + d_cat + self.minimal_ensemble_adapter = None + backbone = dict(backbone) + backbone_type = backbone.pop('type', 'MLP') + self.backbone = lib.deep.make_module(backbone_type, d_in=d_flat, **backbone) + + if arch_type != 'plain': + assert k is not None + first_adapter_init = ( + None + if arch_type == 'tabm-packed' + else 'normal' + if arch_type in ('tabm-mini-normal', 'tabm-normal') + # For other arch_types, the initialization depends + # on the presense of num_embeddings. + else 'random-signs' + if num_embeddings is None + else 'normal' + ) + + if arch_type in ('tabm', 'tabm-normal'): + # Like BatchEnsemble, but all multiplicative adapters, + # except for the very first one, are initialized with ones. + assert first_adapter_init is not None + + use_rla = rla_rank > 1 or rla_additive + if not use_rla: + # Baseline path: rank-1 multiplicative adapter, identical + # to upstream TabM. This branch is kept exact so the rank-1 + # RLA exact-recovery test compares against the original. + lib.deep.make_efficient_ensemble( + self.backbone, + lib.deep.LinearEfficientEnsemble, + k=k, + ensemble_scaling_in=True, + ensemble_scaling_out=True, + ensemble_bias=True, + scaling_init='ones', + ) + _init_first_adapter( + _get_first_ensemble_layer(self.backbone).r, # type: ignore[code] + first_adapter_init, + first_adapter_sections, + ) + else: + # RLA path: rank-r (or additive) adapters. + if rla_first_only: + # All layers rank-1 baseline; replace only the first + # with the rank-r class. + lib.deep.make_efficient_ensemble( + self.backbone, + lib.deep.LinearEfficientEnsemble, + k=k, + ensemble_scaling_in=True, + ensemble_scaling_out=True, + ensemble_bias=True, + scaling_init='ones', + ) + _replace_first_with_rankr( + self.backbone, + k=k, + rank=rla_rank, + additive=rla_additive, + init_mode=rla_init, base_preserve_noise=rla_base_preserve_noise, + ) + else: + # RLA-uniform: every linear layer is rank-r / additive. + lib.deep.make_efficient_ensemble( + self.backbone, + lib.deep.LinearEfficientEnsembleRankR, + k=k, + rank=rla_rank, + scaling_init='ones', + additive=rla_additive, + init_mode=rla_init, base_preserve_noise=rla_base_preserve_noise, + ) + # First-layer R column initialisation: + # - variance_preserving: each rank column gets section-based + # random signs / normal, then scaled by 1/sqrt(rank). + # - base_preserving: path 0 already holds the proper + # section-based init from the layer constructor; extra + # paths stay zero/tiny so we *do not* overwrite them. + if not rla_additive and rla_init != 'base_preserving': + _init_first_adapter_rankr( + _get_first_rankr_layer(self.backbone).R, # type: ignore[code] + first_adapter_init, + first_adapter_sections, + ) + elif not rla_additive and rla_init == 'base_preserving': + # Apply the section-based init only to path 0 of R. + _init_first_adapter_basepreserve( + _get_first_rankr_layer(self.backbone).R, # type: ignore[code] + first_adapter_init, + first_adapter_sections, + ) + + elif arch_type in ('tabm-mini', 'tabm-mini-normal'): + # MiniEnsemble + assert first_adapter_init is not None + self.minimal_ensemble_adapter = lib.deep.ScaleEnsemble( + k, + d_flat, + init='random-signs' if num_embeddings is None else 'normal', + ) + _init_first_adapter( + self.minimal_ensemble_adapter.weight, # type: ignore[code] + first_adapter_init, + first_adapter_sections, + ) + + elif arch_type == 'tabm-packed': + # Packed ensemble. + # In terms of the Packed Ensembles paper by Laurent et al., + # TabM-packed is PackedEnsemble(alpha=k, M=k, gamma=1). + assert first_adapter_init is None + lib.deep.make_efficient_ensemble(self.backbone, lib.deep.NLinear, n=k) + + else: + raise ValueError(f'Unknown arch_type: {arch_type}') + + # >>> Output + d_block = backbone['d_block'] + d_out = 1 if n_classes is None else n_classes + self.output = ( + nn.Linear(d_block, d_out) + if arch_type == 'plain' + else lib.deep.NLinear(k, d_block, d_out) # type: ignore[code] + ) + + # >>> + self.arch_type = arch_type + self.k = k + self.share_training_batches = share_training_batches + self.mfb_cfg = {} if mfb is None else dict(mfb) + self.mfb_enabled = bool(self.mfb_cfg.get('enabled', False)) + self.mfb_mask_mode = str(self.mfb_cfg.get('mask_mode', 'member_fixed')) + self.mfb_mask_granularity = str(self.mfb_cfg.get('mask_granularity', 'feature_group')) + self.mfb_keep_rate = float(self.mfb_cfg.get('keep_rate', 1.0)) + self.mfb_inverted_scaling = bool(self.mfb_cfg.get('inverted_scaling', True)) + self.mfb_use_soft_mask = bool(self.mfb_cfg.get('use_soft_mask', False)) + self.mfb_mask_strength = float(self.mfb_cfg.get('mask_strength', 1.0)) + self.mfb_warmup_epochs = int(self.mfb_cfg.get('warmup_epochs', 0)) + self.mfb_epoch = 0 + self.mfb_feature_widths = [int(x) for x in first_adapter_sections] + if self.mfb_enabled: + assert self.k is not None + if self.mfb_mask_granularity != 'feature_group': + raise ValueError(f'Unsupported MFB mask_granularity={self.mfb_mask_granularity!r}') + feature_mask, dim_mask, mask_stats = _make_mfb_feature_group_mask( + k=self.k, + feature_widths=self.mfb_feature_widths, + keep_rate=self.mfb_keep_rate, + seed=int(self.mfb_cfg.get('mask_seed', 0)), + anchor_fraction=float(self.mfb_cfg.get('anchor_fraction', 0.0)), + protected_feature_ids=self.mfb_cfg.get('protected_feature_ids'), + ) + self.register_buffer('mfb_fixed_feature_mask', torch.from_numpy(feature_mask), persistent=True) + self.register_buffer('mfb_fixed_dim_mask', torch.from_numpy(dim_mask), persistent=True) + self.mfb_mask_stats = mask_stats + else: + self.register_buffer('mfb_fixed_feature_mask', torch.empty((0, 0), dtype=torch.float32), persistent=True) + self.register_buffer('mfb_fixed_dim_mask', torch.empty((0, 0), dtype=torch.float32), persistent=True) + self.mfb_mask_stats = {'keep_rate_actual': 1.0} + + def set_epoch(self, epoch: int) -> None: + self.mfb_epoch = int(epoch) + + def _current_mfb_mask_strength(self) -> float: + if not self.mfb_use_soft_mask: + return 1.0 + if self.mfb_warmup_epochs <= 0: + return self.mfb_mask_strength + return self.mfb_mask_strength * min(1.0, max(0.0, float(self.mfb_epoch) / float(self.mfb_warmup_epochs))) + + def _sample_mfb_mask(self, device_: torch.device, dtype: torch.dtype) -> Tensor: + assert self.k is not None + mask = (torch.rand((self.k, len(self.mfb_feature_widths)), device=device_) < self.mfb_keep_rate).to(dtype) + zero_rows = mask.sum(dim=1) == 0 + if bool(zero_rows.any()): + zero_indices = torch.where(zero_rows)[0] + random_cols = torch.randint(0, len(self.mfb_feature_widths), size=(len(zero_indices),), device=device_) + mask[zero_rows] = 0.0 + mask[zero_indices, random_cols] = 1.0 + feature_index = torch.tensor( + [feature_id for feature_id, width in enumerate(self.mfb_feature_widths) for _ in range(width)], + device=device_, + dtype=torch.long, + ) + return mask[:, feature_index] + + def _apply_mfb_mask(self, x: Tensor) -> Tensor: + if not self.mfb_enabled or self.mfb_mask_mode == 'none': + return x + if self.mfb_mask_mode == 'member_fixed': + raw_mask = self.mfb_fixed_dim_mask.to(device=x.device, dtype=x.dtype) + elif self.mfb_mask_mode == 'stochastic': + if not self.training: + return x + raw_mask = self._sample_mfb_mask(x.device, x.dtype) + else: + raise ValueError(f'Unknown MFB mask_mode={self.mfb_mask_mode!r}') + if self.mfb_use_soft_mask and self.mfb_mask_mode == 'member_fixed': + alpha = self._current_mfb_mask_strength() + effective_mask = (1.0 - alpha) + alpha * raw_mask + return x * effective_mask.unsqueeze(0) + x = x * raw_mask.unsqueeze(0) + if self.mfb_inverted_scaling and self.mfb_keep_rate < 1.0: + x = x / max(self.mfb_keep_rate, 1e-6) return x - mean = np.nanmean(x['train'], axis=0, keepdims=True); mean = np.nan_to_num(mean, nan=0.0) - std = np.nanstd(x['train'], axis=0, keepdims=True); std = np.nan_to_num(std, nan=0.0) - span = np.nanmax(x['train'], axis=0) - np.nanmin(x['train'], axis=0); span = np.nan_to_num(span, nan=0.0) - keep = (std.reshape(-1) >= 1e-6) & (span >= 1e-6) - if not keep.any(): - return {k: np.zeros((v.shape[0], 0), dtype='float32') for k, v in x.items()} - mean = mean[:, keep]; std = std[:, keep] - return {k: np.nan_to_num((v[:, keep] - mean) / std, nan=0.0, posinf=0.0, neginf=0.0).astype('float32') for k, v in x.items()} - -def cat_cardinalities(x_cat): - if not x_cat: + + def forward( + self, x_num: None | Tensor = None, x_cat: None | Tensor = None + ) -> Tensor: + x = [] + if x_num is not None: + x.append(x_num if self.num_module is None else self.num_module(x_num)) + if x_cat is None: + assert self.cat_module is None + else: + assert self.cat_module is not None + x.append(self.cat_module(x_cat).float()) + x = torch.column_stack([x_.flatten(1, -1) for x_ in x]) + + if self.k is not None: + if self.share_training_batches or not self.training: + # (B, D) -> (B, K, D) + x = x[:, None].expand(-1, self.k, -1) + else: + # (B * K, D) -> (B, K, D) + x = x.reshape(len(x) // self.k, self.k, *x.shape[1:]) + if self.minimal_ensemble_adapter is not None: + x = self.minimal_ensemble_adapter(x) + x = self._apply_mfb_mask(x) + else: + assert self.minimal_ensemble_adapter is None + + x = self.backbone(x) + x = self.output(x) + if self.k is None: + # Adjust the output shape for plain networks to make them compatible + # with the rest of the script (loss, metrics, predictions, ...). + # (B, D_OUT) -> (B, 1, D_OUT) + x = x[:, None] + return x + + def get_member_logits( + self, x_num: None | Tensor = None, x_cat: None | Tensor = None + ) -> Tensor: + """Return per-member logits/predictions with shape (B, K, D_OUT).""" + return self.forward(x_num, x_cat) + + def get_first_batchensemble_r(self) -> None | Tensor: + """Expose the first BatchEnsemble `r` parameter when present.""" + if self.arch_type in ('tabm', 'tabm-normal'): + return _get_first_ensemble_layer(self.backbone).r return None - return [max(int(x_cat[p][:, j].max()) for p in x_cat) + 1 for j in range(x_cat['train'].shape[1])] - -def feature_importance(x_num, y): - if x_num.shape[1] == 0: - return np.zeros((0,), dtype='float32') - yy = y.astype('float32'); yy = (yy - yy.mean()) / (yy.std() + 1e-6) - scores = np.nan_to_num(np.abs((x_num.astype('float32') * yy[:, None]).mean(0)), nan=0.0) - if float(scores.max(initial=0.0)) <= 0: - scores = np.ones_like(scores) - return np.clip(scores / (scores.mean() + 1e-6), 0.25, 4.0).astype('float32') - -class IntegratedTabM(nn.Module): - def __init__(self, cfg, n_num_features, cards, d_out, bins, cf_weights): - super().__init__() - mc = cfg['model'] - num_embeddings = None - if n_num_features and bins is not None: - num_embeddings = rtdl_num_embeddings.PiecewiseLinearEmbeddings(bins, int(mc.get('d_embedding', 16)), activation=True, version='B') - self.model = tabm.TabM( - n_num_features=n_num_features, - cat_cardinalities=cards, - d_out=d_out, - num_embeddings=num_embeddings, - arch_type=mc.get('arch_type', 'tabm'), - k=int(mc.get('k', 16)), - n_blocks=int(mc.get('n_blocks', 2)), - d_block=int(mc.get('d_block', 384)), - dropout=float(mc.get('dropout', 0.1)), - start_scaling_init=mc.get('start_scaling_init', 'random-signs'), - ) - self.rla_enabled = bool(mc.get('rla_enabled', False)) and n_num_features > 0 - self.rla_scale = float(mc.get('rla_scale', 0.05)) - if self.rla_enabled: - rank = int(mc.get('rla_rank', 4)) - self.rla_a = nn.Parameter(torch.randn(n_num_features, rank) * 1e-3) - self.rla_b = nn.Parameter(torch.zeros(rank, n_num_features)) - self.mfb_enabled = bool(mc.get('mfb_enabled', False)) and n_num_features > 0 - self.mfb_keep = float(mc.get('mfb_keep', 0.8)) - self.cf_enabled = bool(mc.get('cf_fisd_enabled', False)) and cf_weights is not None and n_num_features > 0 - self.register_buffer('cf_weights', cf_weights if cf_weights is not None else torch.ones(n_num_features)) - def forward(self, x_num, x_cat): - if x_num is not None and x_num.shape[1] == 0: - x_num = None - if x_num is not None and self.cf_enabled: - x_num = x_num * self.cf_weights[None, :] - if x_num is not None and self.rla_enabled: - x_num = x_num + self.rla_scale * (x_num @ self.rla_a @ self.rla_b) - if self.training and x_num is not None and self.mfb_enabled: - mask = (torch.rand((1, x_num.shape[1]), device=x_num.device) < self.mfb_keep).to(x_num.dtype) / max(self.mfb_keep, 1e-6) - x_num = x_num * mask - return self.model(x_num, x_cat) - -def make_loaders(x_num, x_cat, y, task_type, batch_size): - out = {} - for part in ['train', 'val', 'test']: - xc = torch.as_tensor(x_cat[part], dtype=torch.long) if x_cat is not None else torch.empty((len(y[part]), 0), dtype=torch.long) - ds = TensorDataset(torch.as_tensor(x_num[part], dtype=torch.float32), xc, torch.as_tensor(y[part], dtype=torch.float32)) - out[part] = DataLoader(ds, batch_size=batch_size if part == 'train' else batch_size * 4, shuffle=(part == 'train'), num_workers=2, pin_memory=True) - return out - -def predict(model, loader, device, task_type, y_mean, y_std): - model.eval(); preds=[]; targets=[] - with torch.no_grad(): - for xb, xc, yb in loader: - xb=xb.to(device, non_blocking=True); xc=xc.to(device, non_blocking=True) - out = model(xb, xc if xc.shape[1] else None).mean(1).squeeze(-1) - out = torch.sigmoid(out) if task_type == 'binclass' else out * y_std + y_mean - preds.append(out.float().cpu().numpy()); targets.append(yb.numpy()) - return np.concatenate(preds), np.concatenate(targets) - -def metric(task_type, pred, target): - if task_type == 'binclass': - auc = float(roc_auc_score(target.astype(int), pred)); return {'roc-auc': auc, 'score': auc} - rmse = float(math.sqrt(mean_squared_error(target.astype('float32'), pred.astype('float32')))); return {'rmse': rmse, 'score': -rmse} - -def train_one(config_path: Path, output: Path, force=False): - if output.exists() and not force and (output/'DONE').exists() and (output/'report.json').exists(): - return - output.mkdir(parents=True, exist_ok=True) - cfg=load_toml(config_path); set_seed(int(cfg.get('seed',0))) - device=torch.device('cuda' if torch.cuda.is_available() else 'cpu') - x_num,x_cat,y_raw,info=read_arrays((Path.cwd()/cfg['data']['path']).resolve()) - x_num=standardize_num(x_num); task_type=info['task_type']; y={k:v.copy() for k,v in y_raw.items()} - y_mean=0.0; y_std=1.0 - if task_type == 'regression': - y_mean=float(y['train'].mean()); y_std=float(y['train'].std()+1e-6) - y={k:((v.astype('float32')-y_mean)/y_std).astype('float32') for k,v in y.items()} - cards=cat_cardinalities(x_cat); n_num=x_num['train'].shape[1] - bins = rtdl_num_embeddings.compute_bins(torch.as_tensor(x_num['train'], dtype=torch.float32), n_bins=int(cfg['model'].get('n_bins',48))) if n_num else None - cf = torch.as_tensor(feature_importance(x_num['train'], y_raw['train']), dtype=torch.float32) if cfg['model'].get('cf_fisd_enabled', False) else None - model=IntegratedTabM(cfg,n_num,cards,1,bins,cf).to(device) - loaders=make_loaders(x_num,x_cat,y,task_type,int(cfg.get('batch_size',1024))) - opt=torch.optim.AdamW(model.parameters(), lr=float(cfg['optimizer'].get('lr',3e-4)), weight_decay=float(cfg['optimizer'].get('weight_decay',1e-5))) - loss_fn=nn.BCEWithLogitsLoss() if task_type=='binclass' else nn.MSELoss(); amp=bool(cfg.get('amp',True)) and device.type=='cuda' - rho=float(cfg['model'].get('esam_rho',0.0)) if cfg['model'].get('esam_enabled',False) else 0.0 - best=None; best_score=-1e100; bad=0; history=[]; start=time.time() - for epoch in range(int(cfg.get('n_epochs',20))): - model.train(); losses=[] - for xb,xc,yb in loaders['train']: - xb=xb.to(device,non_blocking=True); xc=xc.to(device,non_blocking=True); yb=yb.to(device,non_blocking=True) - opt.zero_grad(set_to_none=True) - with torch.autocast(device_type='cuda', dtype=torch.bfloat16, enabled=amp): - logits=model(xb, xc if xc.shape[1] else None).squeeze(-1); target=yb[:,None].expand_as(logits); loss=loss_fn(logits,target) - if not torch.isfinite(loss): raise RuntimeError(f'NaN/Inf loss at epoch={epoch}') - loss.backward() - if rho > 0: - grads=[p.grad for p in model.parameters() if p.grad is not None] - norm=torch.norm(torch.stack([g.detach().norm() for g in grads])) if grads else torch.tensor(0.0,device=device); scale=rho/(norm+1e-12); eps=[] - with torch.no_grad(): - for p in model.parameters(): - if p.grad is None: eps.append(None) - else: - e=p.grad*scale; p.add_(e); eps.append(e) - opt.zero_grad(set_to_none=True) - with torch.autocast(device_type='cuda', dtype=torch.bfloat16, enabled=amp): - logits2=model(xb, xc if xc.shape[1] else None).squeeze(-1); loss2=loss_fn(logits2,target) - if not torch.isfinite(loss2): raise RuntimeError(f'NaN/Inf ESAM loss at epoch={epoch}') - loss2.backward() - with torch.no_grad(): - for p,e in zip(model.parameters(),eps): - if e is not None: p.sub_(e) - losses.append(float(loss2.detach().cpu())) + + +class Config(TypedDict): + seed: int + data: KWArgs + bins: NotRequired[KWArgs] + model: KWArgs + head_selection: NotRequired[bool] + optimizer: KWArgs + n_lr_warmup_epochs: NotRequired[int] + batch_size: int + eval_batch_size: NotRequired[int] + patience: int + n_epochs: int + gradient_clipping_norm: NotRequired[float] + parameter_statistics: NotRequired[bool] + # NOTE + # Please, read these notes before using AMP and/or `torch.compile`. + # + # The usage of the following efficiency-related settings depends on the model. + # To learn if a given model can run with AMP and torch.compile on a given task, + # try activating these settings and check if the task metrics are satisfactory. + # The following notes can be helpful. + # + # - For simple architectures, such as MLP or TabM, these settings often + # make models significantly faster without any negative side-effects. + # For a real world task, it is worth to doublecheck that by comparing runs + # with and without AMP and/or torch.compile. + # + # - For more complex architectures, these settings should be used + # with extra caution. For example, some baselines used in this project showed + # worse performance when trained with AMP. For some models, AMP with BF16 hurts + # the performance, but AMP with FP16 works fine. Sometimes, it is the opposite. + # Sometimes, it depends on a dataset. Because of that, all baselines were run + # without AMP and torch.compile to ensure that results are representative. + # + # - AMP usually provides significantly larger speedups than `torch.compile`. + # So, if there are any issues with `torch.compile`, using only AMP will still + # lead to substantially faster models. + # + # - If a training run is already fast (e.g. on small datasets), + # `torch.compile` can make it *slower*, because the compilation itself + # takes some time (in particular, at the beginning of the first epoch, + # and at the beginning of the first evaluation). + # + # - Generally, compared to AMP, `torch.compile` is a younger technology, and a + # model must meet certain requirements to be compatible with `torch.compile`. + # In case of any issues, try updating PyTorch. + amp: NotRequired[bool] # torch.autocast + compile: NotRequired[bool] # torch.compile + use_ncl: NotRequired[bool] + lambda_ncl: NotRequired[float] + ncl_warmup_epochs: NotRequired[int] + ncl_space: NotRequired[Literal['logits', 'probs', 'hybrid']] + use_esam: NotRequired[bool] + esam_rho: NotRequired[float] + esam_eps: NotRequired[float] + esam_adapter_only: NotRequired[bool] + esam_memberwise: NotRequired[bool] + esam_warmup_epochs: NotRequired[int] + esam_start_epoch: NotRequired[int] + esam_end_epoch: NotRequired[int] + esam_log_diagnostics: NotRequired[bool] + esam_diagnostics_every: NotRequired[int] + rla_adapter_lr_multiplier: NotRequired[float] + rla_extra_paths_freeze_fraction: NotRequired[float] + cf_fisd: NotRequired[KWArgs] + + +def main( + config: Config | str | Path, + output: None | str | Path = None, + *, + force: bool = False, +) -> None | lib.JSONDict: + # >>> Start + config, output = lib.check(config, output, config_type=Config) + if not lib.start(output, force=force): + return None + + lib.print_config(config) # type: ignore[code] + delu.random.seed(config['seed']) + device = lib.get_device() + report = lib.create_report(main, config) + + # >>> Data + dataset = lib.data.build_dataset(**config['data']) + if dataset.task.is_regression: + dataset.data['y'], regression_label_stats = lib.data.standardize_labels( + dataset.data['y'] + ) + else: + regression_label_stats = None + + # Convert binary features to categorical features. + if dataset.n_bin_features > 0: + x_bin = dataset.data.pop('x_bin') + # Remove binary features with just one unique value in the training set. + # This must be done, otherwise, the script will fail on one specific dataset + # from the "why" benchmark. + n_bin_features = x_bin['train'].shape[1] + good_bin_idx = [ + i for i in range(n_bin_features) if len(np.unique(x_bin['train'][:, i])) > 1 + ] + if len(good_bin_idx) < n_bin_features: + x_bin = {k: v[:, good_bin_idx] for k, v in x_bin.items()} + + if dataset.n_cat_features == 0: + dataset.data['x_cat'] = { + part: np.zeros((dataset.size(part), 0), dtype=np.int64) + for part in x_bin + } + for part in x_bin: + dataset.data['x_cat'][part] = np.column_stack( + [dataset.data['x_cat'][part], x_bin[part].astype(np.int64)] + ) + del x_bin + dataset = dataset.to_torch(device) + Y_train = dataset.data['y']['train'].to( + torch.long if dataset.task.is_classification else torch.float + ) + + # >>> Model + if 'bins' in config: + # Compute the bins for PiecewiseLinearEncoding and PiecewiseLinearEmbeddings. + compute_bins_kwargs = ( + { + 'y': Y_train.to( + torch.long if dataset.task.is_classification else torch.float + ), + 'regression': dataset.task.is_regression, + 'verbose': True, + } + if 'tree_kwargs' in config['bins'] + else {} + ) + bin_edges = rtdl_num_embeddings.compute_bins( + dataset.data['x_num']['train'], **config['bins'], **compute_bins_kwargs + ) + logger.info(f'Bin counts: {[len(x) - 1 for x in bin_edges]}') + else: + bin_edges = None + model = Model( + n_num_features=dataset.n_num_features, + cat_cardinalities=dataset.compute_cat_cardinalities(), + n_classes=dataset.task.try_compute_n_classes(), + **config['model'], + bins=bin_edges, + ) + report['n_parameters'] = lib.deep.get_n_parameters(model) + logger.info(f'n_parameters = {report["n_parameters"]}') + report['prediction_type'] = 'labels' if dataset.task.is_regression else 'probs' + model.to(device) + if lib.is_dataparallel_available(): + model = nn.DataParallel(model) + + root_model = model.module if isinstance(model, nn.DataParallel) else model + report['mfb'] = { + 'enabled': bool(getattr(root_model, 'mfb_enabled', False)), + 'config': dict(getattr(root_model, 'mfb_cfg', {})), + 'mask_stats': dict(getattr(root_model, 'mfb_mask_stats', {})), + } + + cf_fisd_cfg = config.get('cf_fisd') + if cf_fisd_cfg: + cf_fisd_cat_cards = list(dataset.compute_cat_cardinalities()) + if 'num_embeddings' in config['model']: + cf_fisd_d_emb = int(config['model']['num_embeddings']['d_embedding']) + cf_fisd_d_features = [cf_fisd_d_emb] * dataset.n_num_features + cf_fisd_cat_cards + else: + cf_fisd_d_features = [1] * dataset.n_num_features + cf_fisd_cat_cards + cf_fisd_n_features = dataset.n_num_features + len(cf_fisd_cat_cards) + cf_fisd_teacher_names = tuple(cf_fisd_cfg.get('teacher_names', lib.cf_fisd.DEFAULT_TEACHER_NAMES)) + cf_fisd_teachers = lib.cf_fisd.load_teacher_importances( + cf_fisd_cfg['teacher_dir'], + cf_fisd_cfg['dataset_name'], + n_features=cf_fisd_n_features, + teacher_names=cf_fisd_teacher_names, + ) + cf_fisd_member_groups = ( + lib.cf_fisd.default_member_groups(int(config['model']['k']), cf_fisd_teacher_names) + if cf_fisd_cfg.get('member_groups') is None + else {k: list(v) for k, v in cf_fisd_cfg['member_groups'].items()} + ) + cf_fisd_lambda = float(cf_fisd_cfg.get('lambda', 0.0)) + cf_fisd_variant = str(cf_fisd_cfg.get('variant', 'raw')) + cf_fisd_r1_param = _get_first_adapter_for_cf_fisd(root_model.backbone) + report['cf_fisd'] = { + 'lambda': cf_fisd_lambda, + 'variant': cf_fisd_variant, + 'teacher_names': list(cf_fisd_teacher_names), + 'member_groups': {k: list(v) for k, v in cf_fisd_member_groups.items()}, + 'd_features': list(cf_fisd_d_features), + 'n_features': cf_fisd_n_features, + 'teacher_dir': str(cf_fisd_cfg['teacher_dir']), + } + else: + cf_fisd_lambda = 0.0 + cf_fisd_variant = 'raw' + cf_fisd_d_features = [] + cf_fisd_teachers = {} + cf_fisd_member_groups = {} + cf_fisd_r1_param = None + report['cf_fisd'] = {'lambda': 0.0} + + def compute_cf_fisd_penalty() -> Tensor: + if cf_fisd_r1_param is None or cf_fisd_lambda <= 0.0: + return Y_train.new_zeros((), dtype=torch.float32) + return lib.cf_fisd.cf_fisd_loss( + cf_fisd_r1_param, + cf_fisd_teachers, + cf_fisd_member_groups, + cf_fisd_variant, + cf_fisd_d_features, + ) + + # >>> Training + step = 0 + batch_size = config['batch_size'] + report['epoch_size'] = epoch_size = math.ceil(dataset.size('train') / batch_size) + eval_batch_size = config.get( + 'eval_batch_size', + # With torch.compile, + # the largest possible evaluation batch size is noticeably smaller. + 2048 if config.get('compile', False) else 32768, + ) + chunk_size = None # Currently, not used. + share_training_batches = config['model'].get( + 'share_training_batches', DEFAULT_SHARE_TRAINING_BATCHES + ) + use_ncl = config.get('use_ncl', False) + lambda_ncl = float(config.get('lambda_ncl', 0.0)) + ncl_warmup_epochs = int(config.get('ncl_warmup_epochs', 0)) + ncl_space = config.get('ncl_space', 'logits') + if ncl_space not in ('logits', 'probs', 'hybrid'): + raise ValueError(f'Unknown ncl_space: {ncl_space}') + report['ncl'] = { + 'use_ncl': bool(use_ncl), + 'lambda_ncl': lambda_ncl, + 'ncl_warmup_epochs': ncl_warmup_epochs, + 'ncl_space': ncl_space, + } + use_esam = bool(config.get('use_esam', False)) + esam_rho = float(config.get('esam_rho', 0.01)) + esam_eps = float(config.get('esam_eps', 1e-12)) + esam_adapter_only = bool(config.get('esam_adapter_only', True)) + esam_memberwise = bool(config.get('esam_memberwise', True)) + esam_warmup_epochs = int(config.get('esam_warmup_epochs', 0)) + esam_start_epoch = int(config.get('esam_start_epoch', 0)) + esam_end_epoch = int(config.get('esam_end_epoch', -1)) + esam_log_diagnostics = bool(config.get('esam_log_diagnostics', True)) + esam_diagnostics_every = int(config.get('esam_diagnostics_every', 100)) + report['esam'] = { + 'use_esam': use_esam, + 'esam_rho': esam_rho, + 'esam_eps': esam_eps, + 'esam_adapter_only': esam_adapter_only, + 'esam_memberwise': esam_memberwise, + 'esam_warmup_epochs': esam_warmup_epochs, + 'esam_start_epoch': esam_start_epoch, + 'esam_end_epoch': esam_end_epoch, + 'esam_log_diagnostics': esam_log_diagnostics, + 'esam_diagnostics_every': esam_diagnostics_every, + } + + rla_adapter_lr_multiplier = float(config.get('rla_adapter_lr_multiplier', 1.0)) + rla_custom_groups = [] + if rla_adapter_lr_multiplier != 1.0: + rla_adapter_params = [ + p + for module in model.modules() + if isinstance(module, lib.deep.LinearEfficientEnsembleRankR) + for p in (module.R, module.S) + ] + if rla_adapter_params: + rla_custom_groups.append({ + 'params': rla_adapter_params, + 'lr': config['optimizer']['lr'] * rla_adapter_lr_multiplier, + }) + optimizer_cfg = dict(config['optimizer']) + optimizer_type = optimizer_cfg.pop('type', 'AdamW') + optimizer = lib.deep.make_optimizer( + optimizer_type, + **optimizer_cfg, + params=lib.deep.make_parameter_groups(model, custom_groups=rla_custom_groups), + ) + rla_extra_paths_freeze_fraction = float(config.get('rla_extra_paths_freeze_fraction', 0.0)) + assert 0.0 <= rla_extra_paths_freeze_fraction <= 1.0 + rla_extra_paths_freeze_until_epoch = ( + math.ceil(config['n_epochs'] * rla_extra_paths_freeze_fraction) + if config['n_epochs'] > 0 + else math.ceil(2 * config['patience'] * rla_extra_paths_freeze_fraction) + ) + report['rla_training'] = { + 'adapter_lr_multiplier': rla_adapter_lr_multiplier, + 'extra_paths_freeze_fraction': rla_extra_paths_freeze_fraction, + 'extra_paths_freeze_until_epoch': rla_extra_paths_freeze_until_epoch, + } + + def zero_rla_extra_path_grads() -> None: + if rla_extra_paths_freeze_fraction == 0.0: + return + if step // epoch_size >= rla_extra_paths_freeze_until_epoch: + return + for module in model.modules(): + if isinstance(module, lib.deep.LinearEfficientEnsembleRankR) and module.rank > 1: + if module.R.grad is not None: + module.R.grad[:, :, 1:].zero_() + if module.S.grad is not None: + module.S.grad[:, :, 1:].zero_() + gradient_clipping_norm = config.get('gradient_clipping_norm') + _loss_fn = ( + nn.functional.mse_loss + if dataset.task.is_regression + else nn.functional.cross_entropy + ) + + def loss_fn(y_pred: Tensor, y_true: Tensor) -> Tensor: + return _loss_fn( + y_pred.flatten(0, 1), + ( + y_true.repeat_interleave(y_pred.shape[1]) + if share_training_batches + else y_true + ), + ) + + def ncl_penalty(y_pred: Tensor) -> Tensor: + if y_pred.shape[1] <= 1: + return y_pred.new_zeros(()) + + def corr_penalty(member_outputs: Tensor) -> Tensor: + if member_outputs.ndim == 2: + member_outputs = member_outputs.unsqueeze(-1) + centered = member_outputs - member_outputs.mean(dim=1, keepdim=True) + norms = centered.norm(dim=-1) + denom = torch.einsum('bi,bj->bij', norms, norms).clamp_min(1e-8) + corr = torch.einsum('bid,bjd->bij', centered, centered) / denom + k_members = centered.shape[1] + off_diag = ~torch.eye( + k_members, dtype=torch.bool, device=centered.device + ).unsqueeze(0) + return corr.masked_select(off_diag).mean() + + if not dataset.task.is_classification: + return corr_penalty(y_pred) + + if ncl_space == 'logits': + return corr_penalty(y_pred) + if ncl_space == 'probs': + return corr_penalty(y_pred.softmax(-1)) + # Hybrid penalty: equal weighting between logits-space and probability-space. + return 0.5 * (corr_penalty(y_pred) + corr_penalty(y_pred.softmax(-1))) + + base_model = model.module if isinstance(model, nn.DataParallel) else model + + def get_esam_adapter_params() -> list[tuple[str, nn.Parameter]]: + params: list[tuple[str, nn.Parameter]] = [] + if not esam_adapter_only: + params.extend( + [ + (name, param) + for name, param in base_model.named_parameters() + if param.requires_grad + ] + ) + return params + + for module_name, module in base_model.named_modules(): + if isinstance(module, lib.deep.LinearEfficientEnsembleRankR): + for pname in ['R', 'S', 'bias']: + p = getattr(module, pname) + if p is None or not p.requires_grad: + continue + params.append((f'{module_name}.{pname}', p)) + elif isinstance(module, lib.deep.LinearEfficientEnsemble): + for pname in ['r', 's', 'bias']: + p = getattr(module, pname) + if p is None or not p.requires_grad: + continue + if pname == 'bias' and p.ndim != 2: + # Shared bias is not member-specific. + continue + params.append((f'{module_name}.{pname}', p)) + elif isinstance(module, lib.deep.ScaleEnsemble): + if module.weight.requires_grad: + params.append((f'{module_name}.weight', module.weight)) + + if not params: + logger.warning('ESAM adapter-only selection found no adapter parameters.') + return params + + esam_adapter_named_params = get_esam_adapter_params() + esam_adapter_params = [p for _, p in esam_adapter_named_params] + report['esam']['adapter_parameter_names'] = [ + {'name': name, 'shape': list(param.shape)} + for name, param in esam_adapter_named_params + ] + logger.info( + 'ESAM adapter params: {}', + [f'{name}:{tuple(param.shape)}' for name, param in esam_adapter_named_params], + ) + + def compute_esam_perturbations( + grads: list[None | Tensor], + rho: float, + eps: float, + memberwise: bool, + ) -> tuple[list[Tensor], float, float, bool]: + if not grads: + return [], 0.0, 0.0, True + + grad_norm_sq = 0.0 + fallback_used = False + for param, grad in zip(esam_adapter_params, grads): + if grad is None: + continue + if memberwise and param.ndim >= 2 and param.shape[0] > 1: + member_norms = grad.reshape(param.shape[0], -1).norm(dim=1) + grad_norm_sq += float((member_norms**2).sum().item()) else: - losses.append(float(loss.detach().cpu())) - torch.nn.utils.clip_grad_norm_(model.parameters(), float(cfg.get('gradient_clipping_norm',1.0))); opt.step() - vp,vy=predict(model,loaders['val'],device,task_type,y_mean,y_std); tp,ty=predict(model,loaders['test'],device,task_type,y_mean,y_std) - metrics={'val':metric(task_type,vp,vy),'test':metric(task_type,tp,ty)}; score=metrics['val']['score'] - history.append({'epoch':epoch,'train_loss':float(np.mean(losses)),'val_score':score,'test_score':metrics['test']['score']}) - if score > best_score: best_score=score; best={'metrics':metrics,'best_epoch':epoch}; bad=0 + if memberwise: + fallback_used = True + grad_norm_sq += float((grad.norm() ** 2).item()) + grad_norm = math.sqrt(grad_norm_sq) + + perturbations: list[Tensor] = [] + perturb_norm_sq = 0.0 + for param, grad in zip(esam_adapter_params, grads): + if grad is None: + perturb = torch.zeros_like(param) + elif memberwise and param.ndim >= 2 and param.shape[0] > 1: + member_norms = grad.reshape(param.shape[0], -1).norm(dim=1) + view_shape = (param.shape[0],) + (1,) * (param.ndim - 1) + scale = rho / (member_norms.view(view_shape) + eps) + perturb = grad * scale + else: + perturb = grad * (rho / (grad_norm + eps)) + perturbations.append(perturb) + perturb_norm_sq += float((perturb.norm() ** 2).item()) + perturb_norm = math.sqrt(perturb_norm_sq) + return perturbations, grad_norm, perturb_norm, fallback_used + + def apply_esam_perturbation(perturbations: list[Tensor]) -> None: + with torch.no_grad(): + for param, perturb in zip(esam_adapter_params, perturbations): + param.add_(perturb) + + def restore_esam_perturbation(perturbations: list[Tensor]) -> float: + max_abs_restore = 0.0 + with torch.no_grad(): + for param, perturb in zip(esam_adapter_params, perturbations): + param.sub_(perturb) + if perturb.numel(): + max_abs_restore = max(max_abs_restore, float(perturb.abs().max().item())) + return max_abs_restore + + # The following generator is used only for creating training batches, + # so the random seed fully determines the sequence of training objects. + batch_generator = torch.Generator(device).manual_seed(config['seed']) + timer = delu.tools.Timer() + early_stopping = delu.tools.EarlyStopping(config['patience'], mode='max') + parameter_statistics = config.get('parameter_statistics', config['seed'] == 1) + training_log = [] + esam_diag_records: list[dict[str, Any]] = [] + writer = torch.utils.tensorboard.SummaryWriter(output) # type: ignore[code] + writer_failed = False + run_id = f'{output.parent.name}__{output.name}' + esam_diag_path = (output.parent / '_esam_diagnostics') + esam_diag_file = esam_diag_path / f'{run_id}.jsonl' + if use_esam and esam_log_diagnostics: + esam_diag_path.mkdir(parents=True, exist_ok=True) + esam_diag_file.write_text('') + + def safe_add_scalars(tag: str, values: dict[str, Any], step_: int, walltime: float) -> None: + nonlocal writer, writer_failed + if writer is None or writer_failed: + return + try: + writer.add_scalars(tag, values, step_, walltime) + except OSError as err: + writer_failed = True + logger.warning(f'TensorBoard write disabled due to OS error: {err}') + try: + writer.close() + except Exception: + pass + writer = None + + # Only bfloat16 was tested as amp_dtype. + # However, float16 is supported as a fallback. + # To enable float16, uncomment the two lines below. + amp_dtype = ( + torch.bfloat16 + if config.get('amp', False) + and torch.cuda.is_available() + and torch.cuda.is_bf16_supported() + # else torch.float16 + # if config.get('amp', False) and and torch.cuda.is_available() + else None + ) + amp_enabled = amp_dtype is not None + # For FP16, the gradient scaler must be used. + grad_scaler = torch.cuda.amp.GradScaler() if amp_dtype is torch.float16 else None # type: ignore[code] + logger.info(f'AMP enabled: {amp_enabled}') + + if config.get('compile', False): + # NOTE + # `torch.compile` is intentionally called without the `mode` argument, + # because it caused issues with training. + model = torch.compile(model) + evaluation_mode = torch.no_grad + else: + evaluation_mode = torch.inference_mode + + @torch.autocast(device.type, enabled=amp_enabled, dtype=amp_dtype) # type: ignore[code] + def apply_model(part: PartKey, idx: Tensor) -> Tensor: + return ( + model( + dataset.data['x_num'][part][idx] if 'x_num' in dataset.data else None, + dataset.data['x_cat'][part][idx] if 'x_cat' in dataset.data else None, + ) + .squeeze(-1) # Remove the last dimension for regression predictions. + .float() + ) + + @evaluation_mode() + def evaluate( + parts: list[PartKey], eval_batch_size: int + ) -> tuple[ + dict[PartKey, Any], + dict[PartKey, np.ndarray], + dict[PartKey, np.ndarray], + dict[PartKey, Any], + int, + ]: + def compute_diversity(head_preds: np.ndarray) -> dict[str, float]: + if head_preds.ndim == 2: + members = head_preds[..., None] + else: + members = head_preds + members = members.reshape(members.shape[0], members.shape[1], -1) + if members.shape[1] <= 1: + return { + 'mean_centered_corr': 0.0, + 'mean_pairwise_disagreement': 0.0, + 'member_std': 0.0, + } + + centered = members - members.mean(axis=1, keepdims=True) + norms = np.linalg.norm(centered, axis=-1) + denom = np.einsum('bi,bj->bij', norms, norms) + 1e-8 + corr = np.einsum('bid,bjd->bij', centered, centered) / denom + off_diag = ~np.eye(members.shape[1], dtype=bool) + mean_centered_corr = float(corr[:, off_diag].mean()) + + if dataset.task.is_regression: + pairwise_disagreement = float('nan') + elif dataset.task.is_binclass: + labels = (head_preds > 0.5).astype(np.int64) + pairwise_disagreement = float( + (labels[:, :, None] != labels[:, None, :])[:, off_diag].mean() + ) + else: + labels = head_preds.argmax(-1) + pairwise_disagreement = float( + (labels[:, :, None] != labels[:, None, :])[:, off_diag].mean() + ) + + member_std = float(members.std(axis=1).mean()) + return { + 'mean_centered_corr': mean_centered_corr, + 'mean_pairwise_disagreement': pairwise_disagreement, + 'member_std': member_std, + } + + model.eval() + head_predictions: dict[PartKey, np.ndarray] = {} + for part in parts: + while eval_batch_size: + try: + head_predictions[part] = ( + torch.cat( + [ + apply_model(part, idx) + for idx in torch.arange( + dataset.size(part), device=device + ).split(eval_batch_size) + ] + ) + .cpu() + .numpy() + ) + except RuntimeError as err: + if not lib.is_oom_exception(err): + raise + eval_batch_size //= 2 + logger.warning(f'eval_batch_size = {eval_batch_size}') + else: + break + if not eval_batch_size: + RuntimeError('Not enough memory even for eval_batch_size=1') + if dataset.task.is_regression: + assert regression_label_stats is not None + head_predictions = { + k: v * regression_label_stats.std + regression_label_stats.mean + for k, v in head_predictions.items() + } else: - bad += 1 - if bad >= int(cfg.get('patience',6)): break - report={'dataset':cfg['dataset'],'variant':cfg['variant'],'seed':cfg['seed'],'config_path':str(config_path),'result_path':str(output),'gpu_name':torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'cpu','amp_enabled':amp,'amp_dtype':'bfloat16' if amp else 'fp32','inference_mode':'mean','task_type':task_type,'metric_direction':'lower' if task_type=='regression' else 'higher','config':cfg,'time_seconds':time.time()-start,'history':history,**best} - dump_json(output/'report.json',report); (output/'DONE').write_text('done\n') - -def main(): - p=argparse.ArgumentParser(); p.add_argument('config',type=Path); p.add_argument('output',type=Path); p.add_argument('--force',action='store_true'); a=p.parse_args() - try: train_one(a.config,a.output,a.force) - except Exception as e: - a.output.mkdir(parents=True, exist_ok=True); dump_json(a.output/'report.json', {'failure':repr(e),'config_path':str(a.config),'result_path':str(a.output)}); raise -if __name__ == '__main__': main() + head_predictions = { + k: scipy.special.softmax(v, axis=-1) + for k, v in head_predictions.items() + } + if dataset.task.is_binclass: + head_predictions = {k: v[..., 1] for k, v in head_predictions.items()} + + predictions = {k: v.mean(1) for k, v in head_predictions.items()} + metrics = ( + dataset.task.calculate_metrics(predictions, report['prediction_type']) + if lib.are_valid_predictions(predictions) + else {x: {'score': lib.WORST_SCORE} for x in predictions} + ) + diversity = {part: compute_diversity(head_predictions[part]) for part in parts} + return metrics, predictions, head_predictions, diversity, eval_batch_size + + def save_checkpoint() -> None: + lib.dump_checkpoint( + output, + { + 'step': step, + 'model': model.state_dict(), + 'optimizer': optimizer.state_dict(), + 'batch_generator': batch_generator.get_state(), + 'random_state': delu.random.get_state(), + 'early_stopping': early_stopping, + 'report': report, + 'timer': timer, + 'training_log': training_log, + } + | ( + {} if grad_scaler is None else {'grad_scaler': grad_scaler.state_dict()} + ), + ) + lib.dump_report(output, report) + lib.backup_output(output) + + print() + timer.run() + while config['n_epochs'] == -1 or step // epoch_size < config['n_epochs']: + print(f'[...] {lib.try_get_relative_path(output)} | {timer}') + + model.train() + if hasattr(model, 'set_epoch'): + model.set_epoch(step // epoch_size + 1) + epoch_total_losses = [] + epoch_task_losses = [] + epoch_ncl_losses = [] + epoch_esam_sharpness = [] + epoch_esam_grad_norm = [] + epoch_esam_perturb_norm = [] + batches = ( + torch.randperm( + dataset.size('train'), + generator=batch_generator, + device=device, + ).split(batch_size) + if share_training_batches + else [ + x.transpose(0, 1).flatten() + for x in torch.rand( + (config['model']['k'], dataset.size('train')), + generator=batch_generator, + device=device, + ) + .argsort(dim=1) + .split(batch_size, dim=1) + ] + ) + for batch_idx in tqdm(batches, desc=f'Epoch {step // epoch_size} Step {step}'): + optimizer.zero_grad() + y_pred = apply_model('train', batch_idx) + task_loss = loss_fn(y_pred, Y_train[batch_idx]) + ncl_raw = ( + ncl_penalty(y_pred) + if use_ncl and lambda_ncl > 0.0 and y_pred.shape[1] > 1 + else task_loss.new_zeros(()) + ) + if use_ncl and lambda_ncl > 0.0 and ncl_warmup_epochs > 0: + warmup_factor = min( + 1.0, float(step // epoch_size + 1) / float(ncl_warmup_epochs) + ) + else: + warmup_factor = 1.0 + lambda_effective = lambda_ncl * warmup_factor if use_ncl else 0.0 + cf_fisd_penalty = compute_cf_fisd_penalty() + clean_loss = task_loss + lambda_effective * ncl_raw + cf_fisd_lambda * cf_fisd_penalty + if not torch.isfinite(clean_loss): + report['failure'] = {'reason': 'non_finite_loss', 'step': int(step), 'stage': 'clean'} + lib.dump_report(output, report) + raise RuntimeError(f'Non-finite clean loss at step {step}') + + epoch = step // epoch_size + esam_enabled_now = ( + use_esam + and esam_rho > 0.0 + and bool(esam_adapter_params) + and epoch >= esam_start_epoch + and (esam_end_epoch < 0 or epoch <= esam_end_epoch) + ) + if esam_enabled_now and esam_warmup_epochs > 0: + esam_rho_effective = esam_rho * min( + 1.0, float(epoch - esam_start_epoch + 1) / float(esam_warmup_epochs) + ) + else: + esam_rho_effective = esam_rho + + esam_clean = float(clean_loss.detach().item()) + esam_perturbed = esam_clean + esam_sharpness = 0.0 + esam_grad_norm = 0.0 + esam_perturb_norm = 0.0 + esam_fallback_used = False + esam_restore_max_abs = 0.0 + esam_pending_perturbations = None + loss = clean_loss + + if esam_enabled_now and esam_rho_effective > 0.0: + grads = torch.autograd.grad( + clean_loss, + esam_adapter_params, + retain_graph=False, + create_graph=False, + allow_unused=True, + ) + perturbations, esam_grad_norm, esam_perturb_norm, esam_fallback_used = ( + compute_esam_perturbations( + list(grads), + esam_rho_effective, + esam_eps, + esam_memberwise, + ) + ) + apply_esam_perturbation(perturbations) + esam_pending_perturbations = perturbations + try: + y_pred_perturbed = apply_model('train', batch_idx) + pert_task_loss = loss_fn(y_pred_perturbed, Y_train[batch_idx]) + pert_ncl_raw = ( + ncl_penalty(y_pred_perturbed) + if use_ncl and lambda_ncl > 0.0 and y_pred_perturbed.shape[1] > 1 + else pert_task_loss.new_zeros(()) + ) + pert_cf_fisd_penalty = compute_cf_fisd_penalty() + pert_loss = pert_task_loss + lambda_effective * pert_ncl_raw + cf_fisd_lambda * pert_cf_fisd_penalty + if not torch.isfinite(pert_loss): + report['failure'] = {'reason': 'non_finite_loss', 'step': int(step), 'stage': 'perturbed'} + lib.dump_report(output, report) + raise RuntimeError(f'Non-finite perturbed loss at step {step}') + except Exception: + esam_restore_max_abs = restore_esam_perturbation(perturbations) + raise + loss = pert_loss + esam_perturbed = float(pert_loss.detach().item()) + esam_sharpness = esam_perturbed - esam_clean + + optimizer.zero_grad() + if grad_scaler is None: + loss.backward() + else: + grad_scaler.scale(loss).backward() + + if esam_pending_perturbations is not None: + esam_restore_max_abs = restore_esam_perturbation(esam_pending_perturbations) + + zero_rla_extra_path_grads() + + if parameter_statistics and ( + step % epoch_size == 0 # The first batch of the epoch. + or step // epoch_size == 0 # The first epoch. + ): + for k, v in lib.deep.compute_parameter_stats(model).items(): + safe_add_scalars(k, v, step, timer.elapsed()) + del k, v + + if gradient_clipping_norm is not None: + if grad_scaler is not None: + grad_scaler.unscale_(optimizer) + nn.utils.clip_grad.clip_grad_norm_( + model.parameters(), gradient_clipping_norm + ) + if grad_scaler is None: + optimizer.step() + else: + grad_scaler.step(optimizer) + grad_scaler.update() + + step += 1 + epoch_total_losses.append(loss.detach()) + epoch_task_losses.append(task_loss.detach()) + epoch_ncl_losses.append(ncl_raw.detach()) + if esam_enabled_now: + epoch_esam_sharpness.append(esam_sharpness) + epoch_esam_grad_norm.append(esam_grad_norm) + epoch_esam_perturb_norm.append(esam_perturb_norm) + + if use_esam and esam_log_diagnostics and (step % max(1, esam_diagnostics_every) == 0): + rec = { + 'step': step, + 'epoch': epoch, + 'dataset': str(config['data']['path']), + 'esam_enabled_now': esam_enabled_now, + 'esam_fallback_mode': bool(esam_fallback_used), + 'esam_rho_effective': float(esam_rho_effective), + 'clean_loss': esam_clean, + 'perturbed_loss': esam_perturbed, + 'sharpness_proxy': esam_sharpness, + 'adapter_grad_norm': esam_grad_norm, + 'adapter_perturbation_norm': esam_perturb_norm, + 'restore_max_abs_perturb': esam_restore_max_abs, + 'member_loss_available': bool(y_pred.shape[1] > 1), + # Only populated in a follow-up pass where labels are aligned per member. + 'train_member_loss_mean': None, + } + esam_diag_records.append(rec) + with esam_diag_file.open('a') as f: + f.write(json.dumps(rec) + '\n') + + epoch_total_losses = torch.stack(epoch_total_losses).tolist() + epoch_task_losses = torch.stack(epoch_task_losses).tolist() + epoch_ncl_losses = torch.stack(epoch_ncl_losses).tolist() + mean_esam_sharpness = ( + statistics.mean(epoch_esam_sharpness) if epoch_esam_sharpness else 0.0 + ) + mean_esam_grad_norm = ( + statistics.mean(epoch_esam_grad_norm) if epoch_esam_grad_norm else 0.0 + ) + mean_esam_perturb_norm = ( + statistics.mean(epoch_esam_perturb_norm) if epoch_esam_perturb_norm else 0.0 + ) + mean_loss = statistics.mean(epoch_total_losses) + mean_task_loss = statistics.mean(epoch_task_losses) + mean_ncl_loss = statistics.mean(epoch_ncl_losses) + metrics, predictions, _, diversity, eval_batch_size = evaluate( + ['val', 'test'], eval_batch_size + ) + + training_log.append( + { + 'epoch-losses': epoch_total_losses, + 'epoch-task-losses': epoch_task_losses, + 'epoch-ncl-losses': epoch_ncl_losses, + 'lambda_ncl_effective': lambda_effective, + 'esam_mean_sharpness': mean_esam_sharpness, + 'esam_mean_grad_norm': mean_esam_grad_norm, + 'esam_mean_perturb_norm': mean_esam_perturb_norm, + 'metrics': metrics, + 'diversity': diversity, + 'time': timer.elapsed(), + } + ) + lib.print_metrics(mean_loss, metrics) + safe_add_scalars( + 'loss', + { + 'train_total': mean_loss, + 'train_task': mean_task_loss, + 'train_ncl': mean_ncl_loss, + 'train_esam_sharpness': mean_esam_sharpness, + 'train_esam_grad_norm': mean_esam_grad_norm, + 'train_esam_perturb_norm': mean_esam_perturb_norm, + }, + step, + timer.elapsed(), + ) + for part in metrics: + safe_add_scalars( + 'score', {part: metrics[part]['score']}, step, timer.elapsed() + ) + for part in diversity: + safe_add_scalars( + 'diversity/mean_centered_corr', + {part: diversity[part]['mean_centered_corr']}, + step, + timer.elapsed(), + ) + safe_add_scalars( + 'diversity/member_std', + {part: diversity[part]['member_std']}, + step, + timer.elapsed(), + ) + if not math.isnan(diversity[part]['mean_pairwise_disagreement']): + safe_add_scalars( + 'diversity/mean_pairwise_disagreement', + {part: diversity[part]['mean_pairwise_disagreement']}, + step, + timer.elapsed(), + ) + + if ( + 'metrics' not in report + or metrics['val']['score'] > report['metrics']['val']['score'] + ): + print('🌸 New best epoch! 🌸') + report['best_step'] = step + report['metrics'] = metrics + save_checkpoint() + lib.dump_predictions(output, predictions) + + early_stopping.update(metrics['val']['score']) + if early_stopping.should_stop() or not lib.are_valid_predictions(predictions): + break + + print() + report['time'] = str(timer) + + # >>> + if lib.get_checkpoint_path(output).exists(): + model.load_state_dict(lib.load_checkpoint(output)['model']) + report['metrics'], predictions, head_predictions, report['diversity'], _ = evaluate( + ['train', 'val', 'test'], eval_batch_size + ) + report['chunk_size'] = chunk_size + report['eval_batch_size'] = eval_batch_size + report['esam']['diagnostics_path'] = str(esam_diag_file) if use_esam else None + report['esam']['n_diagnostics_records'] = len(esam_diag_records) + lib.dump_predictions(output, predictions) + lib.dump_summary(output, lib.summarize(report)) + save_checkpoint() + + # >>> Submodel selection (TabM[B] & TabM[G]). + if ( + config.get('head_selection', True) + and head_predictions['train'].shape[1] > 1 + # The following conditions is a hack preventing the head selection during + # the hyperparameter tuning, because bin/tune.py runs training + # outside of the project directory. + and lib.env.get_project_dir() in output.parents + and output.parent.name != 'trials' + ): + if output.parent.name.endswith('-evaluation'): + best_head_output = ( + output.parent.with_name( + output.parent.name.removesuffix('-evaluation') + + '-best-head-evaluation' + ) + / output.name + ) + greedy_heads_output = ( + output.parent.with_name( + output.parent.name.removesuffix('-evaluation') + + '-greedy-heads-evaluation' + ) + / output.name + ) + else: + best_head_output = output.with_name(output.name + '-best-head') + greedy_heads_output = output.with_name(output.name + '-greedy-heads') + for dir_ in [best_head_output, greedy_heads_output]: + if dir_.exists(): + logger.warning(f'Removing the existing output: {dir_}') + shutil.rmtree(dir_) + + prediction_type = ( + lib.PredictionType.PROBS + if dataset.task.is_classification + else lib.PredictionType.LABELS + ) + head_selection_timer = delu.tools.Timer() + head_selection_timer.run() + + # >>> TabM[B]: select the Best submodel. + n_heads = head_predictions['val'].shape[1] + head_val_scores = np.array( + [ + dataset.task.calculate_metrics( + {'val': head_predictions['val'][:, i]}, prediction_type + )['val']['score'] + for i in range(n_heads) + ] + ) + best_head_idx = int(np.argmax(head_val_scores)) + best_head_output.mkdir(parents=True) + lib.finish( + best_head_output, + report + | { + 'heads': [best_head_idx], + 'head_selection_time': str(head_selection_timer), + 'metrics': dataset.task.calculate_metrics( + {k: v[:, best_head_idx] for k, v in head_predictions.items()}, + prediction_type, + ), + }, + ) + + # >>> TabM[G]: Greedily select a powerful subset of submodels. + + # Start with the best head. + greedy_idx = [best_head_idx] + greedy_score = head_val_scores[best_head_idx] + + greedy_mask = [False] * n_heads + greedy_mask[best_head_idx] = True + + while len(greedy_idx) < n_heads: + new_idx = None + new_score = None + + # Iterating through all heads. + for head_idx in range(n_heads): + # If the head is already in greedy_idx, skip it. + if greedy_mask[head_idx]: + continue + + candidate_idx = [*greedy_idx, head_idx] + candidate_score = dataset.task.calculate_metrics( + {'val': head_predictions['val'][:, candidate_idx].mean(1)}, + prediction_type, + )['val']['score'] + if candidate_score > greedy_score and ( + new_score is None or candidate_score > new_score + ): + new_idx = candidate_idx + new_score = candidate_score + + # If no head improves the current greedy score, + # the head selection process is stopped. + if new_idx is None: + break + else: + assert new_score is not None + greedy_score = new_score + greedy_idx = new_idx + + greedy_heads_output.mkdir(parents=True) + lib.finish( + greedy_heads_output, + report + | { + 'heads': greedy_idx, + 'head_selection_time': str(head_selection_timer), + 'metrics': dataset.task.calculate_metrics( + {k: v[:, greedy_idx].mean(1) for k, v in head_predictions.items()}, + prediction_type, + ), + }, + ) + + lib.finish(output, report) + return report + + +if __name__ == '__main__': + lib.configure_libraries() + lib.run(main) diff --git a/tabm_integrated/paper/exp/.gitignore b/tabm_integrated/paper/exp/.gitignore index 9d33728e2..f87762f76 100644 --- a/tabm_integrated/paper/exp/.gitignore +++ b/tabm_integrated/paper/exp/.gitignore @@ -7,5 +7,11 @@ FAILED *.pt *.pth checkpoint* +checkpoint.pt +checkpoint_best.pt +predictions*.npz +summary.json +events.out.tfevents.* cache/ __pycache__/ +_esam_diagnostics/ diff --git a/tabm_integrated/paper/exp/final_integrated_audit.csv b/tabm_integrated/paper/exp/final_integrated_audit.csv index 2685aa3b9..00cd114aa 100644 --- a/tabm_integrated/paper/exp/final_integrated_audit.csv +++ b/tabm_integrated/paper/exp/final_integrated_audit.csv @@ -1,151 +1,841 @@ -dataset,variant,seed,result_path,report_json_present,DONE_present,failure_block_absent,gpu_name,amp_dtype,inference_mode,metric,test_metric,failure -sberbank-housing,baseline_plr,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.546495072614636, -sberbank-housing,baseline_plr,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.510137982108967, -sberbank-housing,baseline_plr,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.460434895463788, -sberbank-housing,rla_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.49868567875769, -sberbank-housing,rla_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.506723180155442, -sberbank-housing,rla_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.506943969685626, -sberbank-housing,esam_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.547765632938749, -sberbank-housing,esam_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.510244699059466, -sberbank-housing,esam_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.461513970249932, -sberbank-housing,mfb_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.527389417691648, -sberbank-housing,mfb_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.535810359671178, -sberbank-housing,mfb_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.513798272472021, -sberbank-housing,cf_fisd_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.541630897512452, -sberbank-housing,cf_fisd_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.531481567373374, -sberbank-housing,cf_fisd_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.533345184614086, -sberbank-housing,all_four_combined,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.529940564415122, -sberbank-housing,all_four_combined,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.5397613458401, -sberbank-housing,all_four_combined,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.537548960274977, -sberbank-housing,all_minus_rla,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.530414333922007, -sberbank-housing,all_minus_rla,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.537306272919873, -sberbank-housing,all_minus_rla,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.537770482423289, -sberbank-housing,all_minus_esam,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.529550814849099, -sberbank-housing,all_minus_esam,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.535907580086148, -sberbank-housing,all_minus_esam,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.53778767500918, -sberbank-housing,all_minus_mfb,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.529854542723736, -sberbank-housing,all_minus_mfb,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.527989037555509, -sberbank-housing,all_minus_mfb,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.53653783950218, -sberbank-housing,all_minus_cf_fisd,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.495833595850131, -sberbank-housing,all_minus_cf_fisd,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.52660642338622, -sberbank-housing,all_minus_cf_fisd,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,11.512635297114862, -ecom-offers,baseline_plr,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5767959687132622, -ecom-offers,baseline_plr,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5763749354641947, -ecom-offers,baseline_plr,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.57621293188747, -ecom-offers,rla_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5766629173713693, -ecom-offers,rla_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5767242343675878, -ecom-offers,rla_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5770335251939862, -ecom-offers,esam_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5766903074687046, -ecom-offers,esam_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5767053647384629, -ecom-offers,esam_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5764039992802762, -ecom-offers,mfb_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5756126061669247, -ecom-offers,mfb_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5758805406428937, -ecom-offers,mfb_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5816249076525681, -ecom-offers,cf_fisd_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5731309058948163, -ecom-offers,cf_fisd_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5730458902809421, -ecom-offers,cf_fisd_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5737641046913308, -ecom-offers,all_four_combined,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.575590289916974, -ecom-offers,all_four_combined,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5770100559377882, -ecom-offers,all_four_combined,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5734697781503159, -ecom-offers,all_minus_rla,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5697199786711201, -ecom-offers,all_minus_rla,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.573474359180514, -ecom-offers,all_minus_rla,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5729159973083131, -ecom-offers,all_minus_esam,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.573248515631535, -ecom-offers,all_minus_esam,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5715625167488105, -ecom-offers,all_minus_esam,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5712034110933558, -ecom-offers,all_minus_mfb,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5743883676894275, -ecom-offers,all_minus_mfb,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5738892213665745, -ecom-offers,all_minus_mfb,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5748292097098053, -ecom-offers,all_minus_cf_fisd,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.574504443183962, -ecom-offers,all_minus_cf_fisd,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5749947559912094, -ecom-offers,all_minus_cf_fisd,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.5807682519060278, -homesite-insurance,baseline_plr,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9527329104991943, -homesite-insurance,baseline_plr,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9530616509300706, -homesite-insurance,baseline_plr,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9538975705174861, -homesite-insurance,rla_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9511202086000653, -homesite-insurance,rla_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9523188529200016, -homesite-insurance,rla_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9524476380801628, -homesite-insurance,esam_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9528578176071593, -homesite-insurance,esam_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9530825030113498, -homesite-insurance,esam_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9538855376826346, -homesite-insurance,mfb_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9503995432741815, -homesite-insurance,mfb_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.94973945288198, -homesite-insurance,mfb_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9499664578687927, -homesite-insurance,cf_fisd_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9543305592735997, -homesite-insurance,cf_fisd_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9539302138104276, -homesite-insurance,cf_fisd_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9548333529692444, -homesite-insurance,all_four_combined,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9488481290971439, -homesite-insurance,all_four_combined,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9496361106437045, -homesite-insurance,all_four_combined,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9502789974647928, -homesite-insurance,all_minus_rla,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.950880361340715, -homesite-insurance,all_minus_rla,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9499504624136281, -homesite-insurance,all_minus_rla,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9505799995534802, -homesite-insurance,all_minus_esam,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9485440704751056, -homesite-insurance,all_minus_esam,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9512055136672225, -homesite-insurance,all_minus_esam,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9485803381158939, -homesite-insurance,all_minus_mfb,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9532358612499129, -homesite-insurance,all_minus_mfb,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9528538308245276, -homesite-insurance,all_minus_mfb,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.953869240198483, -homesite-insurance,all_minus_cf_fisd,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9438768769410195, -homesite-insurance,all_minus_cf_fisd,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.9468154377695126, -homesite-insurance,all_minus_cf_fisd,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,roc-auc,0.947036245121507, -cooking-time,baseline_plr,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5612615058011223, -cooking-time,baseline_plr,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5657847681143955, -cooking-time,baseline_plr,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5666563240272247, -cooking-time,rla_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5699112877582024, -cooking-time,rla_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5449543831564863, -cooking-time,rla_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5677737420490234, -cooking-time,esam_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5611478448009337, -cooking-time,esam_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.563722179453466, -cooking-time,esam_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.56671335830164, -cooking-time,mfb_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5555160998555837, -cooking-time,mfb_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.539276959092014, -cooking-time,mfb_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5386993618023146, -cooking-time,cf_fisd_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.563118185514718, -cooking-time,cf_fisd_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5605115851699063, -cooking-time,cf_fisd_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.566079036110905, -cooking-time,all_four_combined,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5511833212856643, -cooking-time,all_four_combined,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.537508726809674, -cooking-time,all_four_combined,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.543946905739664, -cooking-time,all_minus_rla,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.550683292768776, -cooking-time,all_minus_rla,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5451569174185407, -cooking-time,all_minus_rla,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.544944827487872, -cooking-time,all_minus_esam,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5437589902976687, -cooking-time,all_minus_esam,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.538235385864352, -cooking-time,all_minus_esam,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5476520499692263, -cooking-time,all_minus_mfb,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.572318549397406, -cooking-time,all_minus_mfb,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5472973432410306, -cooking-time,all_minus_mfb,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5601664827274426, -cooking-time,all_minus_cf_fisd,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.546692730518933, -cooking-time,all_minus_cf_fisd,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.5418282337637406, -cooking-time,all_minus_cf_fisd,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,2.547009985219361, -delivery-eta,baseline_plr,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8396023796480883, -delivery-eta,baseline_plr,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8639852605326992, -delivery-eta,baseline_plr,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8590972436481992, -delivery-eta,rla_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8738282038863792, -delivery-eta,rla_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8829804025076913, -delivery-eta,rla_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.88058796129553, -delivery-eta,esam_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8414060273006467, -delivery-eta,esam_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.867077237628412, -delivery-eta,esam_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8608628418981936, -delivery-eta,mfb_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.882313199140896, -delivery-eta,mfb_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8999776362056646, -delivery-eta,mfb_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8822192131865638, -delivery-eta,cf_fisd_only,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.847462746808535, -delivery-eta,cf_fisd_only,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8568220804711988, -delivery-eta,cf_fisd_only,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8737455622217765, -delivery-eta,all_four_combined,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8656861082768745, -delivery-eta,all_four_combined,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8804933819744376, -delivery-eta,all_four_combined,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8965344093393364, -delivery-eta,all_minus_rla,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.860688971411864, -delivery-eta,all_minus_rla,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8698356715330888, -delivery-eta,all_minus_rla,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.9066122992751606, -delivery-eta,all_minus_esam,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8574282929864145, -delivery-eta,all_minus_esam,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8896128569809156, -delivery-eta,all_minus_esam,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.889693353794343, -delivery-eta,all_minus_mfb,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8611355949170485, -delivery-eta,all_minus_mfb,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8741570167185646, -delivery-eta,all_minus_mfb,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.870944405411248, -delivery-eta,all_minus_cf_fisd,0,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.891667480580607, -delivery-eta,all_minus_cf_fisd,1,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8636960381640717, -delivery-eta,all_minus_cf_fisd,2,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2,True,True,True,NVIDIA GeForce RTX 4090,bfloat16,mean,rmse,1.8633819488889196, +wave,dataset,variant,seed,metric,metric_key,direction,validation_metric,validation_score,test_metric,inference_mode,config_path,result_path,failure,best_step +baseline_fidelity,sberbank-housing,baseline_plr,0,RMSE,rmse,lower,0.23452539387100468,-0.23452539387100468,0.23846659721529556,mean,exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0,False,814 +baseline_fidelity,sberbank-housing,baseline_plr,0,RMSE,rmse,lower,0.23955454428788905,-0.23955454428788905,0.26588264760298186,best-head,exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-best-head-evaluation/0,False,814 +baseline_fidelity,sberbank-housing,baseline_plr,0,RMSE,rmse,lower,0.23257941028161666,-0.23257941028161666,0.23549883338311248,greedy-heads,exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-greedy-heads-evaluation/0,False,814 +baseline_fidelity,ecom-offers,baseline_plr,0,AUROC,roc-auc,higher,0.6227054517511603,0.6227054517511603,0.591151981311678,mean,exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0,False,321 +baseline_fidelity,ecom-offers,baseline_plr,0,AUROC,roc-auc,higher,0.6404461738890127,0.6404461738890127,0.6034781946000056,best-head,exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-best-head-evaluation/0,False,321 +baseline_fidelity,ecom-offers,baseline_plr,0,AUROC,roc-auc,higher,0.6413971749638401,0.6413971749638401,0.6040305589800323,greedy-heads,exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-greedy-heads-evaluation/0,False,321 +baseline_fidelity,homesite-insurance,baseline_plr,0,AUROC,roc-auc,higher,0.9583121450834431,0.9583121450834431,0.9626260391126089,mean,exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0,False,1980 +baseline_fidelity,homesite-insurance,baseline_plr,0,AUROC,roc-auc,higher,0.9554763470552776,0.9554763470552776,0.959416932798502,best-head,exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-best-head-evaluation/0,False,1980 +baseline_fidelity,homesite-insurance,baseline_plr,0,AUROC,roc-auc,higher,0.9589474980445032,0.9589474980445032,0.9625170670540112,greedy-heads,exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-greedy-heads-evaluation/0,False,1980 +baseline_fidelity,cooking-time,baseline_plr,0,RMSE,rmse,lower,0.4627596029926704,-0.4627596029926704,0.4802386655626689,mean,exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0,False,23088 +baseline_fidelity,cooking-time,baseline_plr,0,RMSE,rmse,lower,0.4652274815891859,-0.4652274815891859,0.4820660657381116,best-head,exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/cooking-time/baseline_plr-best-head-evaluation/0,False,23088 +baseline_fidelity,cooking-time,baseline_plr,0,RMSE,rmse,lower,0.4625169879783321,-0.4625169879783321,0.47990786370986477,greedy-heads,exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/cooking-time/baseline_plr-greedy-heads-evaluation/0,False,23088 +baseline_fidelity,delivery-eta,baseline_plr,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0,False,2184 +baseline_fidelity,delivery-eta,baseline_plr,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-best-head-evaluation/0,False,2184 +baseline_fidelity,delivery-eta,baseline_plr,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0.toml,exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-greedy-heads-evaluation/0,False,2184 +smoke,sberbank-housing,baseline_plr,0,RMSE,rmse,lower,0.23452539387100468,-0.23452539387100468,0.23846659721529556,mean,exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0,False,814 +smoke,sberbank-housing,rla_only,0,RMSE,rmse,lower,0.2354418316334552,-0.2354418316334552,0.23162903716515967,mean,exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0,False,666 +smoke,sberbank-housing,esam_only,0,RMSE,rmse,lower,0.23533073111501152,-0.23533073111501152,0.24162357572341994,mean,exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0,False,888 +smoke,sberbank-housing,mfb_only,0,RMSE,rmse,lower,0.23520949733542967,-0.23520949733542967,0.23880983570262945,mean,exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0,False,1184 +smoke,sberbank-housing,cf_fisd_only,0,RMSE,rmse,lower,0.23523413233394516,-0.23523413233394516,0.235149605552611,mean,exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0,False,666 +smoke,sberbank-housing,all_four_combined,0,RMSE,rmse,lower,0.2348792849855834,-0.2348792849855834,0.2326272249658597,mean,exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0,False,666 +smoke,sberbank-housing,baseline_plr,0,RMSE,rmse,lower,0.23955454428788905,-0.23955454428788905,0.26588264760298186,best-head,exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/baseline_plr-best-head-evaluation/0,False,814 +smoke,sberbank-housing,baseline_plr,0,RMSE,rmse,lower,0.23257941028161666,-0.23257941028161666,0.23549883338311248,greedy-heads,exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/baseline_plr-greedy-heads-evaluation/0,False,814 +smoke,sberbank-housing,cf_fisd_only,0,RMSE,rmse,lower,0.2388197411165918,-0.2388197411165918,0.2462523116085554,best-head,exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/cf_fisd_only-best-head-evaluation/0,False,666 +smoke,sberbank-housing,cf_fisd_only,0,RMSE,rmse,lower,0.23327114512588593,-0.23327114512588593,0.23367141178994547,greedy-heads,exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/cf_fisd_only-greedy-heads-evaluation/0,False,666 +smoke,sberbank-housing,mfb_only,0,RMSE,rmse,lower,0.24356476580119216,-0.24356476580119216,0.2459548830871388,best-head,exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/mfb_only-best-head-evaluation/0,False,1184 +smoke,sberbank-housing,mfb_only,0,RMSE,rmse,lower,0.233840053737128,-0.233840053737128,0.23548819505307594,greedy-heads,exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/mfb_only-greedy-heads-evaluation/0,False,1184 +smoke,sberbank-housing,esam_only,0,RMSE,rmse,lower,0.2398028724080995,-0.2398028724080995,0.23711371881317633,best-head,exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/esam_only-best-head-evaluation/0,False,888 +smoke,sberbank-housing,esam_only,0,RMSE,rmse,lower,0.23380275644783471,-0.23380275644783471,0.24257603366460162,greedy-heads,exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/esam_only-greedy-heads-evaluation/0,False,888 +smoke,sberbank-housing,rla_only,0,RMSE,rmse,lower,0.23765212789668771,-0.23765212789668771,0.24778267915496135,best-head,exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/rla_only-best-head-evaluation/0,False,666 +smoke,sberbank-housing,rla_only,0,RMSE,rmse,lower,0.2336025622310011,-0.2336025622310011,0.23389241285301454,greedy-heads,exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/rla_only-greedy-heads-evaluation/0,False,666 +smoke,sberbank-housing,all_four_combined,0,RMSE,rmse,lower,0.2376718937442043,-0.2376718937442043,0.2539783962159738,best-head,exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/all_four_combined-best-head-evaluation/0,False,666 +smoke,sberbank-housing,all_four_combined,0,RMSE,rmse,lower,0.23335850745168515,-0.23335850745168515,0.23263890687901315,greedy-heads,exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0.toml,exp/integrated/smoke/sberbank-housing/all_four_combined-greedy-heads-evaluation/0,False,666 +smoke,ecom-offers,baseline_plr,0,AUROC,roc-auc,higher,0.6227054517511603,0.6227054517511603,0.591151981311678,mean,exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0.toml,exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0,False,321 +smoke,ecom-offers,rla_only,0,AUROC,roc-auc,higher,0.6226365549199251,0.6226365549199251,0.5911320299643263,mean,exp/integrated/smoke/ecom-offers/rla_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/rla_only-evaluation/0,False,214 +smoke,ecom-offers,esam_only,0,AUROC,roc-auc,higher,0.6227207494642631,0.6227207494642631,0.5911612394559284,mean,exp/integrated/smoke/ecom-offers/esam_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/esam_only-evaluation/0,False,321 +smoke,ecom-offers,mfb_only,0,AUROC,roc-auc,higher,0.6250090898119413,0.6250090898119413,0.5927112610884333,mean,exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0,False,321 +smoke,ecom-offers,cf_fisd_only,0,AUROC,roc-auc,higher,0.6228570653182646,0.6228570653182646,0.5912083608375949,mean,exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0,False,321 +smoke,ecom-offers,all_four_combined,0,AUROC,roc-auc,higher,0.6244437709503304,0.6244437709503304,0.592244305956134,mean,exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0.toml,exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0,False,321 +smoke,ecom-offers,baseline_plr,0,AUROC,roc-auc,higher,0.6404461738890127,0.6404461738890127,0.6034781946000056,best-head,exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0.toml,exp/integrated/smoke/ecom-offers/baseline_plr-best-head-evaluation/0,False,321 +smoke,ecom-offers,mfb_only,0,AUROC,roc-auc,higher,0.6501781765294031,0.6501781765294031,0.6048261425892079,best-head,exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/mfb_only-best-head-evaluation/0,False,321 +smoke,ecom-offers,cf_fisd_only,0,AUROC,roc-auc,higher,0.640920668337369,0.640920668337369,0.6022998594088651,best-head,exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/cf_fisd_only-best-head-evaluation/0,False,321 +smoke,ecom-offers,mfb_only,0,AUROC,roc-auc,higher,0.6506721330887921,0.6506721330887921,0.6064667434006787,greedy-heads,exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/mfb_only-greedy-heads-evaluation/0,False,321 +smoke,ecom-offers,cf_fisd_only,0,AUROC,roc-auc,higher,0.6418041361448852,0.6418041361448852,0.6039617691395597,greedy-heads,exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/cf_fisd_only-greedy-heads-evaluation/0,False,321 +smoke,ecom-offers,esam_only,0,AUROC,roc-auc,higher,0.6406711951042247,0.6406711951042247,0.6023965414622969,best-head,exp/integrated/smoke/ecom-offers/esam_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/esam_only-best-head-evaluation/0,False,321 +smoke,ecom-offers,esam_only,0,AUROC,roc-auc,higher,0.6415752785241469,0.6415752785241469,0.6040898334194843,greedy-heads,exp/integrated/smoke/ecom-offers/esam_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/esam_only-greedy-heads-evaluation/0,False,321 +smoke,ecom-offers,baseline_plr,0,AUROC,roc-auc,higher,0.6413971749638401,0.6413971749638401,0.6040305589800323,greedy-heads,exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0.toml,exp/integrated/smoke/ecom-offers/baseline_plr-greedy-heads-evaluation/0,False,321 +smoke,ecom-offers,rla_only,0,AUROC,roc-auc,higher,0.6394530055211662,0.6394530055211662,0.6014026252811646,best-head,exp/integrated/smoke/ecom-offers/rla_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/rla_only-best-head-evaluation/0,False,214 +smoke,ecom-offers,rla_only,0,AUROC,roc-auc,higher,0.6394530055211662,0.6394530055211662,0.6014026252811646,greedy-heads,exp/integrated/smoke/ecom-offers/rla_only-evaluation/0.toml,exp/integrated/smoke/ecom-offers/rla_only-greedy-heads-evaluation/0,False,214 +smoke,ecom-offers,all_four_combined,0,AUROC,roc-auc,higher,0.6479868261151002,0.6479868261151002,0.6047175337399694,best-head,exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0.toml,exp/integrated/smoke/ecom-offers/all_four_combined-best-head-evaluation/0,False,321 +smoke,ecom-offers,all_four_combined,0,AUROC,roc-auc,higher,0.6505096994576862,0.6505096994576862,0.6063775930816054,greedy-heads,exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0.toml,exp/integrated/smoke/ecom-offers/all_four_combined-greedy-heads-evaluation/0,False,321 +smoke,homesite-insurance,baseline_plr,0,AUROC,roc-auc,higher,0.9583121450834431,0.9583121450834431,0.9626260391126089,mean,exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0,False,1980 +smoke,homesite-insurance,rla_only,0,AUROC,roc-auc,higher,0.9583231528038466,0.9583231528038466,0.96242593886798,mean,exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0,False,1980 +smoke,homesite-insurance,esam_only,0,AUROC,roc-auc,higher,0.9583120395781356,0.9583120395781356,0.9627483366901237,mean,exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0,False,1980 +smoke,homesite-insurance,mfb_only,0,AUROC,roc-auc,higher,0.9580942942079151,0.9580942942079151,0.9616652728230258,mean,exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0,False,1980 +smoke,homesite-insurance,cf_fisd_only,0,AUROC,roc-auc,higher,0.9589599740471012,0.9589599740471012,0.9628908822909436,mean,exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0,False,2420 +smoke,homesite-insurance,all_four_combined,0,AUROC,roc-auc,higher,0.958474253988237,0.958474253988237,0.962487069534998,mean,exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0,False,1760 +smoke,homesite-insurance,baseline_plr,0,AUROC,roc-auc,higher,0.9554763470552776,0.9554763470552776,0.959416932798502,best-head,exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/baseline_plr-best-head-evaluation/0,False,1980 +smoke,homesite-insurance,baseline_plr,0,AUROC,roc-auc,higher,0.9589474980445032,0.9589474980445032,0.9625170670540112,greedy-heads,exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/baseline_plr-greedy-heads-evaluation/0,False,1980 +smoke,homesite-insurance,mfb_only,0,AUROC,roc-auc,higher,0.9560981250001208,0.9560981250001208,0.9601610718172741,best-head,exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/mfb_only-best-head-evaluation/0,False,1980 +smoke,homesite-insurance,mfb_only,0,AUROC,roc-auc,higher,0.9587921238951639,0.9587921238951639,0.9622146031450255,greedy-heads,exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/mfb_only-greedy-heads-evaluation/0,False,1980 +smoke,homesite-insurance,cf_fisd_only,0,AUROC,roc-auc,higher,0.957813544584969,0.957813544584969,0.961490313471295,best-head,exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/cf_fisd_only-best-head-evaluation/0,False,2420 +smoke,homesite-insurance,cf_fisd_only,0,AUROC,roc-auc,higher,0.9594844321379927,0.9594844321379927,0.9634382071404677,greedy-heads,exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/cf_fisd_only-greedy-heads-evaluation/0,False,2420 +smoke,homesite-insurance,esam_only,0,AUROC,roc-auc,higher,0.9561512908829329,0.9561512908829329,0.9606910602029306,best-head,exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/esam_only-best-head-evaluation/0,False,1980 +smoke,homesite-insurance,esam_only,0,AUROC,roc-auc,higher,0.958991133281215,0.958991133281215,0.9627938464179217,greedy-heads,exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/esam_only-greedy-heads-evaluation/0,False,1980 +smoke,homesite-insurance,rla_only,0,AUROC,roc-auc,higher,0.9557610707114682,0.9557610707114682,0.9604946447119459,best-head,exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/rla_only-best-head-evaluation/0,False,1980 +smoke,homesite-insurance,rla_only,0,AUROC,roc-auc,higher,0.959029757015826,0.959029757015826,0.9624579176971494,greedy-heads,exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/rla_only-greedy-heads-evaluation/0,False,1980 +smoke,homesite-insurance,all_four_combined,0,AUROC,roc-auc,higher,0.9569685349939213,0.9569685349939213,0.9582203301945192,best-head,exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/all_four_combined-best-head-evaluation/0,False,1760 +smoke,homesite-insurance,all_four_combined,0,AUROC,roc-auc,higher,0.9592737644154757,0.9592737644154757,0.9627526496640615,greedy-heads,exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0.toml,exp/integrated/smoke/homesite-insurance/all_four_combined-greedy-heads-evaluation/0,False,1760 +smoke,cooking-time,baseline_plr,0,RMSE,rmse,lower,0.4627596029926704,-0.4627596029926704,0.4802386655626689,mean,exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0.toml,exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0,False,23088 +smoke,cooking-time,rla_only,0,RMSE,rmse,lower,0.4627297681263241,-0.4627297681263241,0.48025858554781287,mean,exp/integrated/smoke/cooking-time/rla_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/rla_only-evaluation/0,False,23088 +smoke,cooking-time,esam_only,0,RMSE,rmse,lower,0.4628077726618535,-0.4628077726618535,0.48022789849772834,mean,exp/integrated/smoke/cooking-time/esam_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/esam_only-evaluation/0,False,23088 +smoke,cooking-time,mfb_only,0,RMSE,rmse,lower,0.46290981075910853,-0.46290981075910853,0.480415480847424,mean,exp/integrated/smoke/cooking-time/mfb_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/mfb_only-evaluation/0,False,23088 +smoke,cooking-time,cf_fisd_only,0,RMSE,rmse,lower,0.4632809116304394,-0.4632809116304394,0.48091704691326154,mean,exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0,False,17982 +smoke,cooking-time,all_four_combined,0,RMSE,rmse,lower,0.463355012524859,-0.463355012524859,0.4809202538385541,mean,exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0.toml,exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0,False,17982 +smoke,cooking-time,baseline_plr,0,RMSE,rmse,lower,0.4652274815891859,-0.4652274815891859,0.4820660657381116,best-head,exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0.toml,exp/integrated/smoke/cooking-time/baseline_plr-best-head-evaluation/0,False,23088 +smoke,cooking-time,baseline_plr,0,RMSE,rmse,lower,0.4625169879783321,-0.4625169879783321,0.47990786370986477,greedy-heads,exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0.toml,exp/integrated/smoke/cooking-time/baseline_plr-greedy-heads-evaluation/0,False,23088 +smoke,cooking-time,cf_fisd_only,0,RMSE,rmse,lower,0.46397138529545745,-0.46397138529545745,0.4813020969034895,best-head,exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/cf_fisd_only-best-head-evaluation/0,False,17982 +smoke,cooking-time,cf_fisd_only,0,RMSE,rmse,lower,0.4629530240908241,-0.4629530240908241,0.480628211999089,greedy-heads,exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/cf_fisd_only-greedy-heads-evaluation/0,False,17982 +smoke,cooking-time,mfb_only,0,RMSE,rmse,lower,0.465403548325863,-0.465403548325863,0.48311284423774803,best-head,exp/integrated/smoke/cooking-time/mfb_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/mfb_only-best-head-evaluation/0,False,23088 +smoke,cooking-time,mfb_only,0,RMSE,rmse,lower,0.4625593198933149,-0.4625593198933149,0.479899744054174,greedy-heads,exp/integrated/smoke/cooking-time/mfb_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/mfb_only-greedy-heads-evaluation/0,False,23088 +smoke,cooking-time,esam_only,0,RMSE,rmse,lower,0.4651283707110853,-0.4651283707110853,0.4823068025931435,best-head,exp/integrated/smoke/cooking-time/esam_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/esam_only-best-head-evaluation/0,False,23088 +smoke,cooking-time,esam_only,0,RMSE,rmse,lower,0.4625398779765968,-0.4625398779765968,0.4797024392153251,greedy-heads,exp/integrated/smoke/cooking-time/esam_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/esam_only-greedy-heads-evaluation/0,False,23088 +smoke,cooking-time,rla_only,0,RMSE,rmse,lower,0.4651183431272534,-0.4651183431272534,0.48213882482447107,best-head,exp/integrated/smoke/cooking-time/rla_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/rla_only-best-head-evaluation/0,False,23088 +smoke,cooking-time,rla_only,0,RMSE,rmse,lower,0.4624375326912171,-0.4624375326912171,0.48004674599435365,greedy-heads,exp/integrated/smoke/cooking-time/rla_only-evaluation/0.toml,exp/integrated/smoke/cooking-time/rla_only-greedy-heads-evaluation/0,False,23088 +smoke,cooking-time,all_four_combined,0,RMSE,rmse,lower,0.46391938568359065,-0.46391938568359065,0.48142889277254225,best-head,exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0.toml,exp/integrated/smoke/cooking-time/all_four_combined-best-head-evaluation/0,False,17982 +smoke,cooking-time,all_four_combined,0,RMSE,rmse,lower,0.4629352081332738,-0.4629352081332738,0.4802514336856013,greedy-heads,exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0.toml,exp/integrated/smoke/cooking-time/all_four_combined-greedy-heads-evaluation/0,False,17982 +smoke,delivery-eta,baseline_plr,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0.toml,exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0,False,2184 +smoke,delivery-eta,rla_only,0,RMSE,rmse,lower,0.5556926485294449,-0.5556926485294449,0.5502675684403805,mean,exp/integrated/smoke/delivery-eta/rla_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/rla_only-evaluation/0,False,1365 +smoke,delivery-eta,esam_only,0,RMSE,rmse,lower,0.5556360918188752,-0.5556360918188752,0.5494649592968895,mean,exp/integrated/smoke/delivery-eta/esam_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/esam_only-evaluation/0,False,2184 +smoke,delivery-eta,mfb_only,0,RMSE,rmse,lower,0.5560026829864069,-0.5560026829864069,0.5501201530906976,mean,exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0,False,2184 +smoke,delivery-eta,cf_fisd_only,0,RMSE,rmse,lower,0.5562691235201772,-0.5562691235201772,0.5515996385557942,mean,exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0,False,3003 +smoke,delivery-eta,all_four_combined,0,RMSE,rmse,lower,0.5560789252392828,-0.5560789252392828,0.5493995976501785,mean,exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0.toml,exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0,False,1638 +smoke,delivery-eta,baseline_plr,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0.toml,exp/integrated/smoke/delivery-eta/baseline_plr-best-head-evaluation/0,False,2184 +smoke,delivery-eta,baseline_plr,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0.toml,exp/integrated/smoke/delivery-eta/baseline_plr-greedy-heads-evaluation/0,False,2184 +smoke,delivery-eta,mfb_only,0,RMSE,rmse,lower,0.5590040394051904,-0.5590040394051904,0.5557457406296488,best-head,exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/mfb_only-best-head-evaluation/0,False,2184 +smoke,delivery-eta,mfb_only,0,RMSE,rmse,lower,0.5555016895512279,-0.5555016895512279,0.5503428181880856,greedy-heads,exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/mfb_only-greedy-heads-evaluation/0,False,2184 +smoke,delivery-eta,cf_fisd_only,0,RMSE,rmse,lower,0.5580564462352832,-0.5580564462352832,0.5528824139554394,best-head,exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/cf_fisd_only-best-head-evaluation/0,False,3003 +smoke,delivery-eta,cf_fisd_only,0,RMSE,rmse,lower,0.5559326756199947,-0.5559326756199947,0.5522105562224877,greedy-heads,exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/cf_fisd_only-greedy-heads-evaluation/0,False,3003 +smoke,delivery-eta,esam_only,0,RMSE,rmse,lower,0.5582863288518123,-0.5582863288518123,0.5560390501031047,best-head,exp/integrated/smoke/delivery-eta/esam_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/esam_only-best-head-evaluation/0,False,2184 +smoke,delivery-eta,esam_only,0,RMSE,rmse,lower,0.5551513290724005,-0.5551513290724005,0.5507488105906536,greedy-heads,exp/integrated/smoke/delivery-eta/esam_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/esam_only-greedy-heads-evaluation/0,False,2184 +smoke,delivery-eta,rla_only,0,RMSE,rmse,lower,0.5573103773832353,-0.5573103773832353,0.5532468973961396,best-head,exp/integrated/smoke/delivery-eta/rla_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/rla_only-best-head-evaluation/0,False,1365 +smoke,delivery-eta,rla_only,0,RMSE,rmse,lower,0.5553051102575359,-0.5553051102575359,0.5508335169889497,greedy-heads,exp/integrated/smoke/delivery-eta/rla_only-evaluation/0.toml,exp/integrated/smoke/delivery-eta/rla_only-greedy-heads-evaluation/0,False,1365 +smoke,delivery-eta,all_four_combined,0,RMSE,rmse,lower,0.557107642501001,-0.557107642501001,0.5520572896128711,best-head,exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0.toml,exp/integrated/smoke/delivery-eta/all_four_combined-best-head-evaluation/0,False,1638 +smoke,delivery-eta,all_four_combined,0,RMSE,rmse,lower,0.5558210261849236,-0.5558210261849236,0.551614874495521,greedy-heads,exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0.toml,exp/integrated/smoke/delivery-eta/all_four_combined-greedy-heads-evaluation/0,False,1638 +sweeps,sberbank-housing,rla_rank2_noise0.001,0,RMSE,rmse,lower,0.2354682854519358,-0.2354682854519358,0.24014251007009368,mean,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0,False,814 +sweeps,sberbank-housing,rla_rank2_noise0.0001,0,RMSE,rmse,lower,0.23622468781228076,-0.23622468781228076,0.23935440604862293,mean,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0,False,962 +sweeps,sberbank-housing,rla_rank2_noise1e-05,0,RMSE,rmse,lower,0.23574409357673579,-0.23574409357673579,0.2418545849757855,mean,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0,False,814 +sweeps,sberbank-housing,rla_rank4_noise0.001,0,RMSE,rmse,lower,0.23544592963840097,-0.23544592963840097,0.23541433043431914,mean,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0,False,888 +sweeps,sberbank-housing,rla_rank4_noise0.0001,0,RMSE,rmse,lower,0.2354418316334552,-0.2354418316334552,0.23162903716515967,mean,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0,False,666 +sweeps,sberbank-housing,rla_rank4_noise1e-05,0,RMSE,rmse,lower,0.23604436658345962,-0.23604436658345962,0.2328903381745147,mean,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0,False,666 +sweeps,sberbank-housing,rla_rank8_noise0.001,0,RMSE,rmse,lower,0.23625085686725053,-0.23625085686725053,0.2355661245169954,mean,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0,False,888 +sweeps,sberbank-housing,rla_rank8_noise0.0001,0,RMSE,rmse,lower,0.23560034396487695,-0.23560034396487695,0.2314192197875428,mean,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0,False,666 +sweeps,sberbank-housing,rla_rank8_noise1e-05,0,RMSE,rmse,lower,0.23553038961691936,-0.23553038961691936,0.2411268296110548,mean,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0,False,814 +sweeps,sberbank-housing,esam_rho0.001,0,RMSE,rmse,lower,0.23467774706155475,-0.23467774706155475,0.23716614057842658,mean,exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0,False,888 +sweeps,sberbank-housing,esam_rho0.0025,0,RMSE,rmse,lower,0.23474623366768377,-0.23474623366768377,0.23936563514587833,mean,exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0,False,814 +sweeps,sberbank-housing,esam_rho0.005,0,RMSE,rmse,lower,0.23533073111501152,-0.23533073111501152,0.24162357572341994,mean,exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0,False,888 +sweeps,sberbank-housing,esam_rho0.01,0,RMSE,rmse,lower,0.2345959019738195,-0.2345959019738195,0.23852799100834732,mean,exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0,False,814 +sweeps,sberbank-housing,mfb_keep0.7,0,RMSE,rmse,lower,0.23535581244592554,-0.23535581244592554,0.23562693018337708,mean,exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0,False,888 +sweeps,sberbank-housing,mfb_keep0.8,0,RMSE,rmse,lower,0.2341279953511937,-0.2341279953511937,0.23402994505001048,mean,exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0,False,888 +sweeps,sberbank-housing,mfb_keep0.9,0,RMSE,rmse,lower,0.23520949733542967,-0.23520949733542967,0.23880983570262945,mean,exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0,False,1184 +sweeps,sberbank-housing,mfb_keep0.95,0,RMSE,rmse,lower,0.2352128391676174,-0.2352128391676174,0.23206813724553327,mean,exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0,False,888 +sweeps,sberbank-housing,cf_fisd_lambda0.05,0,RMSE,rmse,lower,0.2359593327633197,-0.2359593327633197,0.23536697907032925,mean,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0,False,666 +sweeps,sberbank-housing,cf_fisd_lambda0.1,0,RMSE,rmse,lower,0.23523413233394516,-0.23523413233394516,0.235149605552611,mean,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0,False,666 +sweeps,sberbank-housing,cf_fisd_lambda0.2,0,RMSE,rmse,lower,0.23533942163580152,-0.23533942163580152,0.24087782495581203,mean,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0,False,814 +sweeps,sberbank-housing,rla_esam,0,RMSE,rmse,lower,0.23528677495791916,-0.23528677495791916,0.23252163677391782,mean,exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0,False,666 +sweeps,sberbank-housing,rla_mfb,0,RMSE,rmse,lower,0.23551030167646542,-0.23551030167646542,0.23422079576562096,mean,exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0,False,888 +sweeps,sberbank-housing,rla_cf_fisd,0,RMSE,rmse,lower,0.23507577716825093,-0.23507577716825093,0.2401363436403948,mean,exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0,False,888 +sweeps,sberbank-housing,esam_mfb,0,RMSE,rmse,lower,0.23493017564475285,-0.23493017564475285,0.23218164932047,mean,exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0,False,888 +sweeps,sberbank-housing,esam_cf_fisd,0,RMSE,rmse,lower,0.2345315410452097,-0.2345315410452097,0.249503671176673,mean,exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0,False,1406 +sweeps,sberbank-housing,mfb_cf_fisd,0,RMSE,rmse,lower,0.23508488120210985,-0.23508488120210985,0.23568789402876444,mean,exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0,False,888 +sweeps,sberbank-housing,all_four_combined,0,RMSE,rmse,lower,0.2348792849855834,-0.2348792849855834,0.2326272249658597,mean,exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0,False,666 +sweeps,sberbank-housing,all_minus_rla,0,RMSE,rmse,lower,0.23564260539703746,-0.23564260539703746,0.2337545527178023,mean,exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0,False,666 +sweeps,sberbank-housing,all_minus_esam,0,RMSE,rmse,lower,0.23494022877441162,-0.23494022877441162,0.2319230328213383,mean,exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0,False,666 +sweeps,sberbank-housing,all_minus_mfb,0,RMSE,rmse,lower,0.23539867958344296,-0.23539867958344296,0.2332565482846656,mean,exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0,False,666 +sweeps,sberbank-housing,all_minus_cf_fisd,0,RMSE,rmse,lower,0.23469635866718716,-0.23469635866718716,0.23237236681744675,mean,exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0,False,666 +sweeps,sberbank-housing,mfb_keep0.7,0,RMSE,rmse,lower,0.24062365494389312,-0.24062365494389312,0.24557535127152327,best-head,exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-best-head-evaluation/0,False,888 +sweeps,sberbank-housing,mfb_keep0.8,0,RMSE,rmse,lower,0.2380346366558266,-0.2380346366558266,0.2579307429494123,best-head,exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-best-head-evaluation/0,False,888 +sweeps,sberbank-housing,mfb_keep0.7,0,RMSE,rmse,lower,0.23323099362803706,-0.23323099362803706,0.2357129689640159,greedy-heads,exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-greedy-heads-evaluation/0,False,888 +sweeps,sberbank-housing,mfb_keep0.8,0,RMSE,rmse,lower,0.23237596587359466,-0.23237596587359466,0.23064696162155762,greedy-heads,exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-greedy-heads-evaluation/0,False,888 +sweeps,sberbank-housing,mfb_keep0.9,0,RMSE,rmse,lower,0.24356476580119216,-0.24356476580119216,0.2459548830871388,best-head,exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-best-head-evaluation/0,False,1184 +sweeps,sberbank-housing,mfb_keep0.9,0,RMSE,rmse,lower,0.233840053737128,-0.233840053737128,0.23548819505307594,greedy-heads,exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-greedy-heads-evaluation/0,False,1184 +sweeps,sberbank-housing,rla_rank2_noise1e-05,0,RMSE,rmse,lower,0.2389693720816827,-0.2389693720816827,0.24127702155259925,best-head,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-best-head-evaluation/0,False,814 +sweeps,sberbank-housing,rla_rank2_noise1e-05,0,RMSE,rmse,lower,0.23334154528971895,-0.23334154528971895,0.23451478286562477,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-greedy-heads-evaluation/0,False,814 +sweeps,sberbank-housing,rla_rank2_noise0.001,0,RMSE,rmse,lower,0.2370483756746748,-0.2370483756746748,0.23920040609688478,best-head,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-best-head-evaluation/0,False,814 +sweeps,sberbank-housing,rla_rank2_noise0.001,0,RMSE,rmse,lower,0.2328613038082034,-0.2328613038082034,0.23498425768312764,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-greedy-heads-evaluation/0,False,814 +sweeps,sberbank-housing,rla_rank2_noise0.0001,0,RMSE,rmse,lower,0.23946739670755524,-0.23946739670755524,0.25239608328273316,best-head,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-best-head-evaluation/0,False,962 +sweeps,sberbank-housing,rla_rank2_noise0.0001,0,RMSE,rmse,lower,0.23332344829675253,-0.23332344829675253,0.24781019077451777,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-greedy-heads-evaluation/0,False,962 +sweeps,sberbank-housing,esam_rho0.01,0,RMSE,rmse,lower,0.23855772550880586,-0.23855772550880586,0.24435949051951408,best-head,exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.01-best-head-evaluation/0,False,814 +sweeps,sberbank-housing,esam_rho0.01,0,RMSE,rmse,lower,0.23321113093450146,-0.23321113093450146,0.23887208479744804,greedy-heads,exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.01-greedy-heads-evaluation/0,False,814 +sweeps,sberbank-housing,esam_rho0.0025,0,RMSE,rmse,lower,0.2377136850270048,-0.2377136850270048,0.262345673404661,best-head,exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-best-head-evaluation/0,False,814 +sweeps,sberbank-housing,esam_rho0.0025,0,RMSE,rmse,lower,0.23295802298295287,-0.23295802298295287,0.23903366020873654,greedy-heads,exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-greedy-heads-evaluation/0,False,814 +sweeps,sberbank-housing,esam_rho0.005,0,RMSE,rmse,lower,0.2398028724080995,-0.2398028724080995,0.23711371881317633,best-head,exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.005-best-head-evaluation/0,False,888 +sweeps,sberbank-housing,esam_rho0.001,0,RMSE,rmse,lower,0.23949031043262045,-0.23949031043262045,0.24155888192007524,best-head,exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.001-best-head-evaluation/0,False,888 +sweeps,sberbank-housing,esam_rho0.005,0,RMSE,rmse,lower,0.23380275644783471,-0.23380275644783471,0.24257603366460162,greedy-heads,exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.005-greedy-heads-evaluation/0,False,888 +sweeps,sberbank-housing,esam_rho0.001,0,RMSE,rmse,lower,0.23316146280351166,-0.23316146280351166,0.23452524296935331,greedy-heads,exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_rho0.001-greedy-heads-evaluation/0,False,888 +sweeps,sberbank-housing,rla_rank4_noise1e-05,0,RMSE,rmse,lower,0.23804322063719607,-0.23804322063719607,0.24414388272972984,best-head,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-best-head-evaluation/0,False,666 +sweeps,sberbank-housing,rla_rank4_noise1e-05,0,RMSE,rmse,lower,0.2335869733918295,-0.2335869733918295,0.2353936312104005,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-greedy-heads-evaluation/0,False,666 +sweeps,sberbank-housing,rla_rank4_noise0.0001,0,RMSE,rmse,lower,0.23765212789668771,-0.23765212789668771,0.24778267915496135,best-head,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-best-head-evaluation/0,False,666 +sweeps,sberbank-housing,rla_rank4_noise0.0001,0,RMSE,rmse,lower,0.2336025622310011,-0.2336025622310011,0.23389241285301454,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-greedy-heads-evaluation/0,False,666 +sweeps,sberbank-housing,rla_rank4_noise0.001,0,RMSE,rmse,lower,0.23958569850307038,-0.23958569850307038,0.2417253037065434,best-head,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-best-head-evaluation/0,False,888 +sweeps,sberbank-housing,rla_rank4_noise0.001,0,RMSE,rmse,lower,0.23389653800469046,-0.23389653800469046,0.23994098363644276,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-greedy-heads-evaluation/0,False,888 +sweeps,sberbank-housing,mfb_keep0.95,0,RMSE,rmse,lower,0.23835928999137548,-0.23835928999137548,0.27696045729988356,best-head,exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-best-head-evaluation/0,False,888 +sweeps,sberbank-housing,mfb_keep0.95,0,RMSE,rmse,lower,0.23254505067097037,-0.23254505067097037,0.23506758402869707,greedy-heads,exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-greedy-heads-evaluation/0,False,888 +sweeps,sberbank-housing,cf_fisd_lambda0.05,0,RMSE,rmse,lower,0.2394238965987828,-0.2394238965987828,0.23525657161407812,best-head,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-best-head-evaluation/0,False,666 +sweeps,sberbank-housing,cf_fisd_lambda0.05,0,RMSE,rmse,lower,0.23452905519184072,-0.23452905519184072,0.23270925018080982,greedy-heads,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-greedy-heads-evaluation/0,False,666 +sweeps,sberbank-housing,cf_fisd_lambda0.1,0,RMSE,rmse,lower,0.2388197411165918,-0.2388197411165918,0.2462523116085554,best-head,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-best-head-evaluation/0,False,666 +sweeps,sberbank-housing,cf_fisd_lambda0.1,0,RMSE,rmse,lower,0.23327114512588593,-0.23327114512588593,0.23367141178994547,greedy-heads,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-greedy-heads-evaluation/0,False,666 +sweeps,sberbank-housing,cf_fisd_lambda0.2,0,RMSE,rmse,lower,0.23866861156973368,-0.23866861156973368,0.2497089599248454,best-head,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-best-head-evaluation/0,False,814 +sweeps,sberbank-housing,cf_fisd_lambda0.2,0,RMSE,rmse,lower,0.23337569983908155,-0.23337569983908155,0.244512625524316,greedy-heads,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-greedy-heads-evaluation/0,False,814 +sweeps,sberbank-housing,rla_rank8_noise0.0001,0,RMSE,rmse,lower,0.23612384794902308,-0.23612384794902308,0.25131993238610145,best-head,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-best-head-evaluation/0,False,666 +sweeps,sberbank-housing,rla_rank8_noise0.0001,0,RMSE,rmse,lower,0.23331262295778735,-0.23331262295778735,0.2320760028721284,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-greedy-heads-evaluation/0,False,666 +sweeps,sberbank-housing,rla_rank8_noise1e-05,0,RMSE,rmse,lower,0.23899585630204528,-0.23899585630204528,0.25744912519923097,best-head,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-best-head-evaluation/0,False,814 +sweeps,sberbank-housing,rla_rank8_noise1e-05,0,RMSE,rmse,lower,0.23346652550110514,-0.23346652550110514,0.24465339979644968,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-greedy-heads-evaluation/0,False,814 +sweeps,sberbank-housing,rla_rank8_noise0.001,0,RMSE,rmse,lower,0.239120553017687,-0.239120553017687,0.23826554512424344,best-head,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-best-head-evaluation/0,False,888 +sweeps,sberbank-housing,rla_rank8_noise0.001,0,RMSE,rmse,lower,0.23353637226653048,-0.23353637226653048,0.23842460978700958,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-greedy-heads-evaluation/0,False,888 +sweeps,sberbank-housing,mfb_cf_fisd,0,RMSE,rmse,lower,0.23765498863853576,-0.23765498863853576,0.2536206620016002,best-head,exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-best-head-evaluation/0,False,888 +sweeps,sberbank-housing,mfb_cf_fisd,0,RMSE,rmse,lower,0.23320900639668657,-0.23320900639668657,0.23362227205150313,greedy-heads,exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-greedy-heads-evaluation/0,False,888 +sweeps,sberbank-housing,rla_mfb,0,RMSE,rmse,lower,0.2399809438744672,-0.2399809438744672,0.24194702392020134,best-head,exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_mfb-best-head-evaluation/0,False,888 +sweeps,sberbank-housing,rla_mfb,0,RMSE,rmse,lower,0.23358161473727912,-0.23358161473727912,0.2330478525093287,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_mfb-greedy-heads-evaluation/0,False,888 +sweeps,sberbank-housing,esam_mfb,0,RMSE,rmse,lower,0.23962393015909736,-0.23962393015909736,0.24513925399849143,best-head,exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_mfb-best-head-evaluation/0,False,888 +sweeps,sberbank-housing,esam_mfb,0,RMSE,rmse,lower,0.2336977870530739,-0.2336977870530739,0.23066386352840565,greedy-heads,exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_mfb-greedy-heads-evaluation/0,False,888 +sweeps,sberbank-housing,rla_cf_fisd,0,RMSE,rmse,lower,0.23788056532412807,-0.23788056532412807,0.24833117735063384,best-head,exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-best-head-evaluation/0,False,888 +sweeps,sberbank-housing,rla_cf_fisd,0,RMSE,rmse,lower,0.233603375533187,-0.233603375533187,0.23596160620261294,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-greedy-heads-evaluation/0,False,888 +sweeps,sberbank-housing,all_minus_rla,0,RMSE,rmse,lower,0.2383111872393019,-0.2383111872393019,0.2360172512727007,best-head,exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_rla-best-head-evaluation/0,False,666 +sweeps,sberbank-housing,all_minus_rla,0,RMSE,rmse,lower,0.23419188657775022,-0.23419188657775022,0.23442550750747457,greedy-heads,exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_rla-greedy-heads-evaluation/0,False,666 +sweeps,sberbank-housing,all_minus_esam,0,RMSE,rmse,lower,0.23762339316571812,-0.23762339316571812,0.2548487995005296,best-head,exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_esam-best-head-evaluation/0,False,666 +sweeps,sberbank-housing,all_minus_esam,0,RMSE,rmse,lower,0.23366970594362974,-0.23366970594362974,0.23132655964186158,greedy-heads,exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_esam-greedy-heads-evaluation/0,False,666 +sweeps,sberbank-housing,rla_esam,0,RMSE,rmse,lower,0.2398484083586863,-0.2398484083586863,0.2374880282622196,best-head,exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_esam-best-head-evaluation/0,False,666 +sweeps,sberbank-housing,rla_esam,0,RMSE,rmse,lower,0.2336232925794531,-0.2336232925794531,0.233028022164673,greedy-heads,exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/rla_esam-greedy-heads-evaluation/0,False,666 +sweeps,sberbank-housing,esam_cf_fisd,0,RMSE,rmse,lower,0.2404271956933659,-0.2404271956933659,0.25751796428274565,best-head,exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-best-head-evaluation/0,False,1406 +sweeps,sberbank-housing,esam_cf_fisd,0,RMSE,rmse,lower,0.23329227219621285,-0.23329227219621285,0.23856120001818734,greedy-heads,exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-greedy-heads-evaluation/0,False,1406 +sweeps,sberbank-housing,all_four_combined,0,RMSE,rmse,lower,0.2376718937442043,-0.2376718937442043,0.2539783962159738,best-head,exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_four_combined-best-head-evaluation/0,False,666 +sweeps,sberbank-housing,all_four_combined,0,RMSE,rmse,lower,0.23335850745168515,-0.23335850745168515,0.23263890687901315,greedy-heads,exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_four_combined-greedy-heads-evaluation/0,False,666 +sweeps,sberbank-housing,all_minus_cf_fisd,0,RMSE,rmse,lower,0.23808479027162685,-0.23808479027162685,0.27950919024280946,best-head,exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-best-head-evaluation/0,False,666 +sweeps,sberbank-housing,all_minus_cf_fisd,0,RMSE,rmse,lower,0.23321538793887137,-0.23321538793887137,0.23519795893846665,greedy-heads,exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-greedy-heads-evaluation/0,False,666 +sweeps,sberbank-housing,all_minus_mfb,0,RMSE,rmse,lower,0.23769862439897374,-0.23769862439897374,0.24291600363173604,best-head,exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_mfb-best-head-evaluation/0,False,666 +sweeps,sberbank-housing,all_minus_mfb,0,RMSE,rmse,lower,0.23349088968550719,-0.23349088968550719,0.23959467004131907,greedy-heads,exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/sberbank-housing/all_minus_mfb-greedy-heads-evaluation/0,False,666 +sweeps,ecom-offers,rla_rank2_noise0.001,0,AUROC,roc-auc,higher,0.6226233688882503,0.6226233688882503,0.5911217830862241,mean,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank2_noise0.0001,0,AUROC,roc-auc,higher,0.6227001707049361,0.6227001707049361,0.5911765632808944,mean,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank2_noise1e-05,0,AUROC,roc-auc,higher,0.6226529287429354,0.6226529287429354,0.5911485904814773,mean,exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank4_noise0.001,0,AUROC,roc-auc,higher,0.622598916870318,0.622598916870318,0.5911027119909538,mean,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank4_noise0.0001,0,AUROC,roc-auc,higher,0.6226365549199251,0.6226365549199251,0.5911320299643263,mean,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank4_noise1e-05,0,AUROC,roc-auc,higher,0.6226645595746754,0.6226645595746754,0.5911489283247057,mean,exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank8_noise0.001,0,AUROC,roc-auc,higher,0.6225384372823319,0.6225384372823319,0.591035974005685,mean,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank8_noise0.0001,0,AUROC,roc-auc,higher,0.6225693459597093,0.6225693459597093,0.5910522493707531,mean,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank8_noise1e-05,0,AUROC,roc-auc,higher,0.6226251931156621,0.6226251931156621,0.5910938288836818,mean,exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0,False,214 +sweeps,ecom-offers,esam_rho0.001,0,AUROC,roc-auc,higher,0.6227874903903754,0.6227874903903754,0.5911993816464689,mean,exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0,False,321 +sweeps,ecom-offers,esam_rho0.0025,0,AUROC,roc-auc,higher,0.6227333347907498,0.6227333347907498,0.5911601949314514,mean,exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0,False,321 +sweeps,ecom-offers,esam_rho0.005,0,AUROC,roc-auc,higher,0.6227207494642631,0.6227207494642631,0.5911612394559284,mean,exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0,False,321 +sweeps,ecom-offers,esam_rho0.01,0,AUROC,roc-auc,higher,0.6227573335158101,0.6227573335158101,0.5911827002496308,mean,exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0,False,321 +sweeps,ecom-offers,mfb_keep0.7,0,AUROC,roc-auc,higher,0.6291199291618959,0.6291199291618959,0.5961977102213372,mean,exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0,False,428 +sweeps,ecom-offers,mfb_keep0.8,0,AUROC,roc-auc,higher,0.6293880905914133,0.6293880905914133,0.5962685705138886,mean,exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0,False,321 +sweeps,ecom-offers,mfb_keep0.9,0,AUROC,roc-auc,higher,0.6250090898119413,0.6250090898119413,0.5927112610884333,mean,exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0,False,321 +sweeps,ecom-offers,mfb_keep0.95,0,AUROC,roc-auc,higher,0.6252559501617828,0.6252559501617828,0.5929062493223765,mean,exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0,False,321 +sweeps,ecom-offers,cf_fisd_lambda0.05,0,AUROC,roc-auc,higher,0.6228186386127663,0.6228186386127663,0.5911736900637132,mean,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0,False,321 +sweeps,ecom-offers,cf_fisd_lambda0.1,0,AUROC,roc-auc,higher,0.6228570653182646,0.6228570653182646,0.5912083608375949,mean,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0,False,321 +sweeps,ecom-offers,cf_fisd_lambda0.2,0,AUROC,roc-auc,higher,0.622913421046708,0.622913421046708,0.5913362546461658,mean,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0,False,214 +sweeps,ecom-offers,rla_esam,0,AUROC,roc-auc,higher,0.6225317005394864,0.6225317005394864,0.5910259812848737,mean,exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0,False,214 +sweeps,ecom-offers,rla_mfb,0,AUROC,roc-auc,higher,0.6244884295114916,0.6244884295114916,0.5923593741199417,mean,exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0,False,321 +sweeps,ecom-offers,rla_cf_fisd,0,AUROC,roc-auc,higher,0.6227441474841354,0.6227441474841354,0.5911932818714825,mean,exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0,False,214 +sweeps,ecom-offers,esam_mfb,0,AUROC,roc-auc,higher,0.6249124647240485,0.6249124647240485,0.5926966501436739,mean,exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0,False,321 +sweeps,ecom-offers,esam_cf_fisd,0,AUROC,roc-auc,higher,0.6227702320934685,0.6227702320934685,0.5911624823470715,mean,exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0,False,321 +sweeps,ecom-offers,mfb_cf_fisd,0,AUROC,roc-auc,higher,0.6250550271749447,0.6250550271749447,0.592670158895295,mean,exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0,False,321 +sweeps,ecom-offers,all_four_combined,0,AUROC,roc-auc,higher,0.6244437709503304,0.6244437709503304,0.592244305956134,mean,exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0,False,321 +sweeps,ecom-offers,all_minus_rla,0,AUROC,roc-auc,higher,0.6249365260993432,0.6249365260993432,0.5925964470819166,mean,exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0,False,321 +sweeps,ecom-offers,all_minus_esam,0,AUROC,roc-auc,higher,0.6245232151327823,0.6245232151327823,0.5923207050179439,mean,exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0,False,321 +sweeps,ecom-offers,all_minus_mfb,0,AUROC,roc-auc,higher,0.6226455728683622,0.6226455728683622,0.5910976319446108,mean,exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0,False,214 +sweeps,ecom-offers,all_minus_cf_fisd,0,AUROC,roc-auc,higher,0.6244116424360375,0.6244116424360375,0.5923170352346188,mean,exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0,False,321 +sweeps,ecom-offers,rla_rank2_noise0.001,0,AUROC,roc-auc,higher,0.6395828610304147,0.6395828610304147,0.601475038412775,best-head,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-best-head-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank2_noise0.0001,0,AUROC,roc-auc,higher,0.6396541901648656,0.6396541901648656,0.6015418321886687,best-head,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-best-head-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank2_noise0.001,0,AUROC,roc-auc,higher,0.6395828610304147,0.6395828610304147,0.601475038412775,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank2_noise0.0001,0,AUROC,roc-auc,higher,0.6396541901648656,0.6396541901648656,0.6015418321886687,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank2_noise1e-05,0,AUROC,roc-auc,higher,0.6395464280764918,0.6395464280764918,0.6014523595237629,best-head,exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-best-head-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank2_noise1e-05,0,AUROC,roc-auc,higher,0.6395464280764918,0.6395464280764918,0.6014523595237629,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank4_noise0.001,0,AUROC,roc-auc,higher,0.6394467257565004,0.6394467257565004,0.6014832923257777,best-head,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-best-head-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank4_noise0.0001,0,AUROC,roc-auc,higher,0.6394530055211662,0.6394530055211662,0.6014026252811646,best-head,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-best-head-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank4_noise0.001,0,AUROC,roc-auc,higher,0.6394467257565004,0.6394467257565004,0.6014832923257777,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank4_noise1e-05,0,AUROC,roc-auc,higher,0.6396100364908871,0.6396100364908871,0.6014864444960832,best-head,exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-best-head-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank4_noise0.0001,0,AUROC,roc-auc,higher,0.6394530055211662,0.6394530055211662,0.6014026252811646,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank4_noise1e-05,0,AUROC,roc-auc,higher,0.6396100364908871,0.6396100364908871,0.6014864444960832,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,mfb_keep0.7,0,AUROC,roc-auc,higher,0.6547032702847417,0.6547032702847417,0.6118564029158264,best-head,exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.7-best-head-evaluation/0,False,428 +sweeps,ecom-offers,mfb_keep0.7,0,AUROC,roc-auc,higher,0.6547032702847417,0.6547032702847417,0.6118564029158264,greedy-heads,exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.7-greedy-heads-evaluation/0,False,428 +sweeps,ecom-offers,mfb_keep0.8,0,AUROC,roc-auc,higher,0.652377048657135,0.652377048657135,0.6066780069996902,best-head,exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.8-best-head-evaluation/0,False,321 +sweeps,ecom-offers,esam_rho0.001,0,AUROC,roc-auc,higher,0.6410241296714208,0.6410241296714208,0.6024966050474919,best-head,exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.001-best-head-evaluation/0,False,321 +sweeps,ecom-offers,mfb_keep0.9,0,AUROC,roc-auc,higher,0.6501781765294031,0.6501781765294031,0.6048261425892079,best-head,exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.9-best-head-evaluation/0,False,321 +sweeps,ecom-offers,esam_rho0.0025,0,AUROC,roc-auc,higher,0.64045684285539,0.64045684285539,0.6022469172053364,best-head,exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.0025-best-head-evaluation/0,False,321 +sweeps,ecom-offers,esam_rho0.005,0,AUROC,roc-auc,higher,0.6406711951042247,0.6406711951042247,0.6023965414622969,best-head,exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.005-best-head-evaluation/0,False,321 +sweeps,ecom-offers,mfb_keep0.8,0,AUROC,roc-auc,higher,0.6540790454616192,0.6540790454616192,0.6102977368359448,greedy-heads,exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.8-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,esam_rho0.01,0,AUROC,roc-auc,higher,0.6407133660825518,0.6407133660825518,0.6023744607727624,best-head,exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.01-best-head-evaluation/0,False,321 +sweeps,ecom-offers,esam_rho0.001,0,AUROC,roc-auc,higher,0.641845068859757,0.641845068859757,0.6041622806453656,greedy-heads,exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.001-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,mfb_keep0.9,0,AUROC,roc-auc,higher,0.6506721330887921,0.6506721330887921,0.6064667434006787,greedy-heads,exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.9-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,esam_rho0.0025,0,AUROC,roc-auc,higher,0.641480466607742,0.641480466607742,0.6039784195416061,greedy-heads,exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.0025-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,esam_rho0.005,0,AUROC,roc-auc,higher,0.6415752785241469,0.6415752785241469,0.6040898334194843,greedy-heads,exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.005-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,esam_rho0.01,0,AUROC,roc-auc,higher,0.6415860912175324,0.6415860912175324,0.6040412707800087,greedy-heads,exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_rho0.01-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,mfb_keep0.95,0,AUROC,roc-auc,higher,0.6503189442353507,0.6503189442353507,0.6050413611236272,best-head,exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.95-best-head-evaluation/0,False,321 +sweeps,ecom-offers,rla_rank8_noise0.001,0,AUROC,roc-auc,higher,0.6392187010153481,0.6392187010153481,0.6012738729168664,best-head,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-best-head-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank8_noise0.0001,0,AUROC,roc-auc,higher,0.639398116545268,0.639398116545268,0.6013710911801927,best-head,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-best-head-evaluation/0,False,214 +sweeps,ecom-offers,cf_fisd_lambda0.2,0,AUROC,roc-auc,higher,0.638077626500147,0.638077626500147,0.6005513130366958,best-head,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-best-head-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank8_noise0.001,0,AUROC,roc-auc,higher,0.6392187010153481,0.6392187010153481,0.6012738729168664,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,cf_fisd_lambda0.05,0,AUROC,roc-auc,higher,0.6410202011331968,0.6410202011331968,0.6024701571918213,best-head,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-best-head-evaluation/0,False,321 +sweeps,ecom-offers,rla_rank8_noise0.0001,0,AUROC,roc-auc,higher,0.639398116545268,0.639398116545268,0.6013710911801927,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,cf_fisd_lambda0.1,0,AUROC,roc-auc,higher,0.640920668337369,0.640920668337369,0.6022998594088651,best-head,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-best-head-evaluation/0,False,321 +sweeps,ecom-offers,mfb_keep0.95,0,AUROC,roc-auc,higher,0.6505655760961024,0.6505655760961024,0.6064442039882287,greedy-heads,exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_keep0.95-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,cf_fisd_lambda0.05,0,AUROC,roc-auc,higher,0.6419149054445072,0.6419149054445072,0.6041224275423289,greedy-heads,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,cf_fisd_lambda0.1,0,AUROC,roc-auc,higher,0.6418041361448852,0.6418041361448852,0.6039617691395597,greedy-heads,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,rla_rank8_noise1e-05,0,AUROC,roc-auc,higher,0.6393903110631306,0.6393903110631306,0.6013715932958167,best-head,exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-best-head-evaluation/0,False,214 +sweeps,ecom-offers,cf_fisd_lambda0.2,0,AUROC,roc-auc,higher,0.6387008784019832,0.6387008784019832,0.6023730040175572,greedy-heads,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,rla_rank8_noise1e-05,0,AUROC,roc-auc,higher,0.6393903110631306,0.6393903110631306,0.6013715932958167,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,mfb_cf_fisd,0,AUROC,roc-auc,higher,0.6477890245840551,0.6477890245840551,0.6045088303107439,best-head,exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-best-head-evaluation/0,False,321 +sweeps,ecom-offers,mfb_cf_fisd,0,AUROC,roc-auc,higher,0.6506783465179156,0.6506783465179156,0.6065618416202598,greedy-heads,exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,esam_mfb,0,AUROC,roc-auc,higher,0.6500013075472437,0.6500013075472437,0.6046018053871005,best-head,exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_mfb-best-head-evaluation/0,False,321 +sweeps,ecom-offers,rla_mfb,0,AUROC,roc-auc,higher,0.6500073735640511,0.6500073735640511,0.6048362127969982,best-head,exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_mfb-best-head-evaluation/0,False,321 +sweeps,ecom-offers,rla_cf_fisd,0,AUROC,roc-auc,higher,0.6377017988002615,0.6377017988002615,0.6003639247256524,best-head,exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_cf_fisd-best-head-evaluation/0,False,214 +sweeps,ecom-offers,esam_mfb,0,AUROC,roc-auc,higher,0.650368091501537,0.650368091501537,0.6061843343570334,greedy-heads,exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_mfb-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,rla_mfb,0,AUROC,roc-auc,higher,0.6505360715210358,0.6505360715210358,0.6064993654188358,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_mfb-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,esam_cf_fisd,0,AUROC,roc-auc,higher,0.6410485079831951,0.6410485079831951,0.6024310820580536,best-head,exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_cf_fisd-best-head-evaluation/0,False,321 +sweeps,ecom-offers,rla_cf_fisd,0,AUROC,roc-auc,higher,0.6386263098818863,0.6386263098818863,0.6021922454924368,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_cf_fisd-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,esam_cf_fisd,0,AUROC,roc-auc,higher,0.6417659932080955,0.6417659932080955,0.6040036864957168,greedy-heads,exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/esam_cf_fisd-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,all_minus_rla,0,AUROC,roc-auc,higher,0.6475121989956593,0.6475121989956593,0.6043243524106316,best-head,exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_rla-best-head-evaluation/0,False,321 +sweeps,ecom-offers,rla_esam,0,AUROC,roc-auc,higher,0.6394659925462143,0.6394659925462143,0.601268355843962,best-head,exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_esam-best-head-evaluation/0,False,214 +sweeps,ecom-offers,rla_esam,0,AUROC,roc-auc,higher,0.6394659925462143,0.6394659925462143,0.601268355843962,greedy-heads,exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/rla_esam-greedy-heads-evaluation/0,False,214 +sweeps,ecom-offers,all_minus_rla,0,AUROC,roc-auc,higher,0.6504789566191646,0.6504789566191646,0.6063598206681031,greedy-heads,exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_rla-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,all_minus_esam,0,AUROC,roc-auc,higher,0.6476721782117019,0.6476721782117019,0.6040083543113315,best-head,exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_esam-best-head-evaluation/0,False,321 +sweeps,ecom-offers,all_minus_esam,0,AUROC,roc-auc,higher,0.6504598446123828,0.6504598446123828,0.6064818006704371,greedy-heads,exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_esam-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,all_four_combined,0,AUROC,roc-auc,higher,0.6479868261151002,0.6479868261151002,0.6047175337399694,best-head,exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_four_combined-best-head-evaluation/0,False,321 +sweeps,ecom-offers,all_minus_mfb,0,AUROC,roc-auc,higher,0.637709740638791,0.637709740638791,0.6004564813723224,best-head,exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_mfb-best-head-evaluation/0,False,214 +sweeps,ecom-offers,all_four_combined,0,AUROC,roc-auc,higher,0.6505096994576862,0.6505096994576862,0.6063775930816054,greedy-heads,exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_four_combined-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,all_minus_cf_fisd,0,AUROC,roc-auc,higher,0.6498655113216565,0.6498655113216565,0.6047694345185003,best-head,exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-best-head-evaluation/0,False,321 +sweeps,ecom-offers,all_minus_cf_fisd,0,AUROC,roc-auc,higher,0.6505072081895442,0.6505072081895442,0.6062618182365493,greedy-heads,exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-greedy-heads-evaluation/0,False,321 +sweeps,ecom-offers,all_minus_mfb,0,AUROC,roc-auc,higher,0.6386704340734016,0.6386704340734016,0.6021744792778928,greedy-heads,exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/ecom-offers/all_minus_mfb-greedy-heads-evaluation/0,False,214 +sweeps,homesite-insurance,rla_rank2_noise0.001,0,AUROC,roc-auc,higher,0.958447068787368,0.958447068787368,0.9625908587761747,mean,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_rank2_noise0.0001,0,AUROC,roc-auc,higher,0.9582141658213201,0.9582141658213201,0.962170313614336,mean,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_rank2_noise1e-05,0,AUROC,roc-auc,higher,0.9583854185194205,0.9583854185194205,0.9623873274822505,mean,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_rank4_noise0.001,0,AUROC,roc-auc,higher,0.9581143050478821,0.9581143050478821,0.9619551240015043,mean,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0,False,1100 +sweeps,homesite-insurance,rla_rank4_noise0.0001,0,AUROC,roc-auc,higher,0.9583231528038466,0.9583231528038466,0.96242593886798,mean,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_rank4_noise1e-05,0,AUROC,roc-auc,higher,0.9581626616470993,0.9581626616470993,0.9620964494052155,mean,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_rank8_noise0.001,0,AUROC,roc-auc,higher,0.9580761824634811,0.9580761824634811,0.962025146401907,mean,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_rank8_noise0.0001,0,AUROC,roc-auc,higher,0.9582609925935802,0.9582609925935802,0.9623517001429152,mean,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0,False,1100 +sweeps,homesite-insurance,rla_rank8_noise1e-05,0,AUROC,roc-auc,higher,0.958187182838957,0.958187182838957,0.9624786972914716,mean,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0,False,1760 +sweeps,homesite-insurance,esam_rho0.001,0,AUROC,roc-auc,higher,0.958366515485181,0.958366515485181,0.9625169220800972,mean,exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0,False,1980 +sweeps,homesite-insurance,esam_rho0.0025,0,AUROC,roc-auc,higher,0.9581563313286563,0.9581563313286563,0.96205585670933,mean,exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0,False,1760 +sweeps,homesite-insurance,esam_rho0.005,0,AUROC,roc-auc,higher,0.9583120395781356,0.9583120395781356,0.9627483366901237,mean,exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0,False,1980 +sweeps,homesite-insurance,esam_rho0.01,0,AUROC,roc-auc,higher,0.9580370399944422,0.9580370399944422,0.9621106810110944,mean,exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0,False,1980 +sweeps,homesite-insurance,mfb_keep0.7,0,AUROC,roc-auc,higher,0.9574455596570353,0.9574455596570353,0.9607387445394369,mean,exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0,False,1980 +sweeps,homesite-insurance,mfb_keep0.8,0,AUROC,roc-auc,higher,0.9580661506671708,0.9580661506671708,0.9619981087669689,mean,exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0,False,1760 +sweeps,homesite-insurance,mfb_keep0.9,0,AUROC,roc-auc,higher,0.9580942942079151,0.9580942942079151,0.9616652728230258,mean,exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0,False,1980 +sweeps,homesite-insurance,mfb_keep0.95,0,AUROC,roc-auc,higher,0.9584246137411132,0.9584246137411132,0.962318960200698,mean,exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0,False,1980 +sweeps,homesite-insurance,cf_fisd_lambda0.05,0,AUROC,roc-auc,higher,0.9587449454385457,0.9587449454385457,0.9627875642149869,mean,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0,False,1980 +sweeps,homesite-insurance,cf_fisd_lambda0.1,0,AUROC,roc-auc,higher,0.9589599740471012,0.9589599740471012,0.9628908822909436,mean,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0,False,2420 +sweeps,homesite-insurance,cf_fisd_lambda0.2,0,AUROC,roc-auc,higher,0.9587151577734279,0.9587151577734279,0.9626366101271623,mean,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0,False,2200 +sweeps,homesite-insurance,rla_esam,0,AUROC,roc-auc,higher,0.9581143402163179,0.9581143402163179,0.962225790298713,mean,exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_mfb,0,AUROC,roc-auc,higher,0.9581138918187615,0.9581138918187615,0.9616453389098677,mean,exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_cf_fisd,0,AUROC,roc-auc,higher,0.9584762673811861,0.9584762673811861,0.9621948866927381,mean,exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0,False,1100 +sweeps,homesite-insurance,esam_mfb,0,AUROC,roc-auc,higher,0.9580893002900325,0.9580893002900325,0.9618081204528327,mean,exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0,False,1980 +sweeps,homesite-insurance,esam_cf_fisd,0,AUROC,roc-auc,higher,0.9589333075806602,0.9589333075806602,0.9627584002959787,mean,exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0,False,2200 +sweeps,homesite-insurance,mfb_cf_fisd,0,AUROC,roc-auc,higher,0.9589238560635404,0.9589238560635404,0.9624954901031625,mean,exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0,False,1980 +sweeps,homesite-insurance,all_four_combined,0,AUROC,roc-auc,higher,0.958474253988237,0.958474253988237,0.962487069534998,mean,exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0,False,1760 +sweeps,homesite-insurance,all_minus_rla,0,AUROC,roc-auc,higher,0.9586943908120914,0.9586943908120914,0.9623596978704976,mean,exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0,False,1980 +sweeps,homesite-insurance,all_minus_esam,0,AUROC,roc-auc,higher,0.95846243739381,0.95846243739381,0.9621719566520266,mean,exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0,False,1100 +sweeps,homesite-insurance,all_minus_mfb,0,AUROC,roc-auc,higher,0.9585091762449806,0.9585091762449806,0.9622794548091665,mean,exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0,False,1100 +sweeps,homesite-insurance,all_minus_cf_fisd,0,AUROC,roc-auc,higher,0.9580700807398709,0.9580700807398709,0.9619997518046595,mean,exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_rank2_noise0.0001,0,AUROC,roc-auc,higher,0.9559529233208351,0.9559529233208351,0.9601378276664159,best-head,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-best-head-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_rank2_noise1e-05,0,AUROC,roc-auc,higher,0.9558571420859494,0.9558571420859494,0.958774227194806,best-head,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-best-head-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_rank2_noise0.001,0,AUROC,roc-auc,higher,0.9553321388843028,0.9553321388843028,0.9596838539362689,best-head,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_rank2_noise0.001,0,AUROC,roc-auc,higher,0.9591161746546817,0.9591161746546817,0.9623163265079293,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_rank2_noise0.0001,0,AUROC,roc-auc,higher,0.9589703663198782,0.9589703663198782,0.9625895781732687,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-greedy-heads-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_rank2_noise1e-05,0,AUROC,roc-auc,higher,0.9591292133522524,0.9591292133522524,0.9626703044809795,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-greedy-heads-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_rank4_noise0.001,0,AUROC,roc-auc,higher,0.9576809947504603,0.9576809947504603,0.9611084763444616,best-head,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-best-head-evaluation/0,False,1100 +sweeps,homesite-insurance,rla_rank4_noise0.001,0,AUROC,roc-auc,higher,0.9589014449778305,0.9589014449778305,0.9626523760402965,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-greedy-heads-evaluation/0,False,1100 +sweeps,homesite-insurance,esam_rho0.0025,0,AUROC,roc-auc,higher,0.9561005955827354,0.9561005955827354,0.9607496659076156,best-head,exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-best-head-evaluation/0,False,1760 +sweeps,homesite-insurance,mfb_keep0.7,0,AUROC,roc-auc,higher,0.9540607823461205,0.9540607823461205,0.9604495578247303,best-head,exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,esam_rho0.001,0,AUROC,roc-auc,higher,0.9561281764285071,0.9561281764285071,0.9607344074031798,best-head,exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.001-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,mfb_keep0.8,0,AUROC,roc-auc,higher,0.9557633918282306,0.9557633918282306,0.9594055402651032,best-head,exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-best-head-evaluation/0,False,1760 +sweeps,homesite-insurance,mfb_keep0.7,0,AUROC,roc-auc,higher,0.9588480241238587,0.9588480241238587,0.9630162122394903,greedy-heads,exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,esam_rho0.0025,0,AUROC,roc-auc,higher,0.95870368407125,0.95870368407125,0.962465432178352,greedy-heads,exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-greedy-heads-evaluation/0,False,1760 +sweeps,homesite-insurance,mfb_keep0.9,0,AUROC,roc-auc,higher,0.9560981250001208,0.9560981250001208,0.9601610718172741,best-head,exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,esam_rho0.001,0,AUROC,roc-auc,higher,0.9589766350935588,0.9589766350935588,0.962915757398333,greedy-heads,exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.001-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,mfb_keep0.8,0,AUROC,roc-auc,higher,0.9587700557017027,0.9587700557017027,0.9623207965369405,greedy-heads,exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-greedy-heads-evaluation/0,False,1760 +sweeps,homesite-insurance,esam_rho0.005,0,AUROC,roc-auc,higher,0.9561512908829329,0.9561512908829329,0.9606910602029306,best-head,exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.005-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,mfb_keep0.9,0,AUROC,roc-auc,higher,0.9587921238951639,0.9587921238951639,0.9622146031450255,greedy-heads,exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,esam_rho0.01,0,AUROC,roc-auc,higher,0.9554589123032324,0.9554589123032324,0.9585579261153012,best-head,exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.01-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,esam_rho0.005,0,AUROC,roc-auc,higher,0.958991133281215,0.958991133281215,0.9627938464179217,greedy-heads,exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.005-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,esam_rho0.01,0,AUROC,roc-auc,higher,0.9587733087820138,0.9587733087820138,0.9626104060922291,greedy-heads,exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_rho0.01-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_rank4_noise0.0001,0,AUROC,roc-auc,higher,0.9557610707114682,0.9557610707114682,0.9604946447119459,best-head,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_rank4_noise0.0001,0,AUROC,roc-auc,higher,0.959029757015826,0.959029757015826,0.9624579176971494,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,mfb_keep0.95,0,AUROC,roc-auc,higher,0.9563975490624737,0.9563975490624737,0.9593891219693567,best-head,exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_rank4_noise1e-05,0,AUROC,roc-auc,higher,0.9560453459701026,0.9560453459701026,0.9600924508313724,best-head,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-best-head-evaluation/0,False,1760 +sweeps,homesite-insurance,mfb_keep0.95,0,AUROC,roc-auc,higher,0.9592957095194116,0.9592957095194116,0.9620218603265258,greedy-heads,exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_rank4_noise1e-05,0,AUROC,roc-auc,higher,0.9591329763748824,0.9591329763748824,0.962370812537228,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-greedy-heads-evaluation/0,False,1760 +sweeps,homesite-insurance,cf_fisd_lambda0.05,0,AUROC,roc-auc,higher,0.9577627701557911,0.9577627701557911,0.9622852537657216,best-head,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,cf_fisd_lambda0.05,0,AUROC,roc-auc,higher,0.9594017599375494,0.9594017599375494,0.9632589952205967,greedy-heads,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,cf_fisd_lambda0.1,0,AUROC,roc-auc,higher,0.957813544584969,0.957813544584969,0.961490313471295,best-head,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-best-head-evaluation/0,False,2420 +sweeps,homesite-insurance,cf_fisd_lambda0.1,0,AUROC,roc-auc,higher,0.9594844321379927,0.9594844321379927,0.9634382071404677,greedy-heads,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-greedy-heads-evaluation/0,False,2420 +sweeps,homesite-insurance,cf_fisd_lambda0.2,0,AUROC,roc-auc,higher,0.9576897428988642,0.9576897428988642,0.9612223171103336,best-head,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-best-head-evaluation/0,False,2200 +sweeps,homesite-insurance,cf_fisd_lambda0.2,0,AUROC,roc-auc,higher,0.9594008895187635,0.9594008895187635,0.9626779880984148,greedy-heads,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-greedy-heads-evaluation/0,False,2200 +sweeps,homesite-insurance,mfb_cf_fisd,0,AUROC,roc-auc,higher,0.9576967765860231,0.9576967765860231,0.9617315500639867,best-head,exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,mfb_cf_fisd,0,AUROC,roc-auc,higher,0.9597570226838343,0.9597570226838343,0.9633062567165207,greedy-heads,exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_rank8_noise0.0001,0,AUROC,roc-auc,higher,0.9580729733437149,0.9580729733437149,0.9613215034297445,best-head,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-best-head-evaluation/0,False,1100 +sweeps,homesite-insurance,rla_rank8_noise0.0001,0,AUROC,roc-auc,higher,0.9588554974164651,0.9588554974164651,0.9627966975715613,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-greedy-heads-evaluation/0,False,1100 +sweeps,homesite-insurance,esam_mfb,0,AUROC,roc-auc,higher,0.9551748656394307,0.9551748656394307,0.9597148541915197,best-head,exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_mfb-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,esam_mfb,0,AUROC,roc-auc,higher,0.9588471624971818,0.9588471624971818,0.962094141903753,greedy-heads,exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_mfb-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_rank8_noise0.001,0,AUROC,roc-auc,higher,0.9565663135937417,0.9565663135937417,0.9586702567362371,best-head,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-best-head-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_rank8_noise0.001,0,AUROC,roc-auc,higher,0.9588770556676071,0.9588770556676071,0.9621469244895637,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-greedy-heads-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_rank8_noise1e-05,0,AUROC,roc-auc,higher,0.9560226359526883,0.9560226359526883,0.9594298354768371,best-head,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-best-head-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_rank8_noise1e-05,0,AUROC,roc-auc,higher,0.9588073342436448,0.9588073342436448,0.9625660923992208,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-greedy-heads-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_cf_fisd,0,AUROC,roc-auc,higher,0.9580215219221477,0.9580215219221477,0.9612981867919294,best-head,exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-best-head-evaluation/0,False,1100 +sweeps,homesite-insurance,esam_cf_fisd,0,AUROC,roc-auc,higher,0.9573917959108148,0.9573917959108148,0.9606087512633268,best-head,exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-best-head-evaluation/0,False,2200 +sweeps,homesite-insurance,rla_cf_fisd,0,AUROC,roc-auc,higher,0.9589731006657612,0.9589731006657612,0.9623105275513741,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-greedy-heads-evaluation/0,False,1100 +sweeps,homesite-insurance,esam_cf_fisd,0,AUROC,roc-auc,higher,0.9595481045909983,0.9595481045909983,0.9631976229303886,greedy-heads,exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-greedy-heads-evaluation/0,False,2200 +sweeps,homesite-insurance,rla_mfb,0,AUROC,roc-auc,higher,0.9554584990741118,0.9554584990741118,0.9598635611836792,best-head,exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_mfb-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,all_minus_rla,0,AUROC,roc-auc,higher,0.9573359044742286,0.9573359044742286,0.9616503767533748,best-head,exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_rla-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,rla_mfb,0,AUROC,roc-auc,higher,0.9589065795694565,0.9589065795694565,0.961628401124263,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_mfb-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,all_minus_rla,0,AUROC,roc-auc,higher,0.9594920636885601,0.9594920636885601,0.9630834801355295,greedy-heads,exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_rla-greedy-heads-evaluation/0,False,1980 +sweeps,homesite-insurance,all_minus_esam,0,AUROC,roc-auc,higher,0.9578664730808395,0.9578664730808395,0.9618357621457454,best-head,exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_esam-best-head-evaluation/0,False,1100 +sweeps,homesite-insurance,all_minus_esam,0,AUROC,roc-auc,higher,0.959329480009883,0.959329480009883,0.9630054358452254,greedy-heads,exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_esam-greedy-heads-evaluation/0,False,1100 +sweeps,homesite-insurance,rla_esam,0,AUROC,roc-auc,higher,0.9561241320583908,0.9561241320583908,0.960971584726283,best-head,exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_esam-best-head-evaluation/0,False,1760 +sweeps,homesite-insurance,rla_esam,0,AUROC,roc-auc,higher,0.9589296236870106,0.9589296236870106,0.9624745655349262,greedy-heads,exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/rla_esam-greedy-heads-evaluation/0,False,1760 +sweeps,homesite-insurance,all_minus_mfb,0,AUROC,roc-auc,higher,0.9578946605821286,0.9578946605821286,0.9613487585255535,best-head,exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_mfb-best-head-evaluation/0,False,1100 +sweeps,homesite-insurance,all_minus_mfb,0,AUROC,roc-auc,higher,0.9590512800985321,0.9590512800985321,0.9625182147641627,greedy-heads,exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_mfb-greedy-heads-evaluation/0,False,1100 +sweeps,homesite-insurance,all_four_combined,0,AUROC,roc-auc,higher,0.9569685349939213,0.9569685349939213,0.9582203301945192,best-head,exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_four_combined-best-head-evaluation/0,False,1760 +sweeps,homesite-insurance,all_four_combined,0,AUROC,roc-auc,higher,0.9592737644154757,0.9592737644154757,0.9627526496640615,greedy-heads,exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_four_combined-greedy-heads-evaluation/0,False,1760 +sweeps,homesite-insurance,all_minus_cf_fisd,0,AUROC,roc-auc,higher,0.9560637742304589,0.9560637742304589,0.9602624689888717,best-head,exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-best-head-evaluation/0,False,1980 +sweeps,homesite-insurance,all_minus_cf_fisd,0,AUROC,roc-auc,higher,0.9587104803714672,0.9587104803714672,0.9619912225060596,greedy-heads,exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-greedy-heads-evaluation/0,False,1980 +sweeps,cooking-time,rla_rank2_noise0.001,0,RMSE,rmse,lower,0.462741360963445,-0.462741360963445,0.4802330803717953,mean,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank2_noise0.0001,0,RMSE,rmse,lower,0.4627581378606596,-0.4627581378606596,0.4802975698569232,mean,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank2_noise1e-05,0,RMSE,rmse,lower,0.46274171518445095,-0.46274171518445095,0.48025462954174786,mean,exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank4_noise0.001,0,RMSE,rmse,lower,0.46272482498106277,-0.46272482498106277,0.4802920784266626,mean,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank4_noise0.0001,0,RMSE,rmse,lower,0.4627297681263241,-0.4627297681263241,0.48025858554781287,mean,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank4_noise1e-05,0,RMSE,rmse,lower,0.4627757191386438,-0.4627757191386438,0.4802501150008962,mean,exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank8_noise0.001,0,RMSE,rmse,lower,0.4627485258356125,-0.4627485258356125,0.4803784913673949,mean,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank8_noise0.0001,0,RMSE,rmse,lower,0.4627781501968618,-0.4627781501968618,0.4803483238337491,mean,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank8_noise1e-05,0,RMSE,rmse,lower,0.46274024999580365,-0.46274024999580365,0.48030301468765013,mean,exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.001,0,RMSE,rmse,lower,0.46276101981921397,-0.46276101981921397,0.4801992110036178,mean,exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.0025,0,RMSE,rmse,lower,0.4627563346148938,-0.4627563346148938,0.4802735094812688,mean,exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.005,0,RMSE,rmse,lower,0.4628077726618535,-0.4628077726618535,0.48022789849772834,mean,exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.01,0,RMSE,rmse,lower,0.462806194991412,-0.462806194991412,0.4802449022823802,mean,exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.7,0,RMSE,rmse,lower,0.4641162243965573,-0.4641162243965573,0.4815966695140456,mean,exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.8,0,RMSE,rmse,lower,0.4634085546239816,-0.4634085546239816,0.4808915146871742,mean,exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.9,0,RMSE,rmse,lower,0.46290981075910853,-0.46290981075910853,0.480415480847424,mean,exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.95,0,RMSE,rmse,lower,0.46283832689198484,-0.46283832689198484,0.48036774294670015,mean,exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0,False,23088 +sweeps,cooking-time,cf_fisd_lambda0.05,0,RMSE,rmse,lower,0.4632019090821483,-0.4632019090821483,0.4808496810468957,mean,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0,False,17982 +sweeps,cooking-time,cf_fisd_lambda0.1,0,RMSE,rmse,lower,0.4632809116304394,-0.4632809116304394,0.48091704691326154,mean,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0,False,17982 +sweeps,cooking-time,cf_fisd_lambda0.2,0,RMSE,rmse,lower,0.4633341406854049,-0.4633341406854049,0.4808806847716492,mean,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0,False,15096 +sweeps,cooking-time,rla_esam,0,RMSE,rmse,lower,0.4627353230636857,-0.4627353230636857,0.4803077303885999,mean,exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0,False,23088 +sweeps,cooking-time,rla_mfb,0,RMSE,rmse,lower,0.4628964355361592,-0.4628964355361592,0.4804232040775468,mean,exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0,False,23088 +sweeps,cooking-time,rla_cf_fisd,0,RMSE,rmse,lower,0.46314488441622087,-0.46314488441622087,0.48088551875549623,mean,exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0,False,17982 +sweeps,cooking-time,esam_mfb,0,RMSE,rmse,lower,0.4629259699594227,-0.4629259699594227,0.4803877661460669,mean,exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0,False,23088 +sweeps,cooking-time,esam_cf_fisd,0,RMSE,rmse,lower,0.4632865725335359,-0.4632865725335359,0.48093585437297304,mean,exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0,False,17982 +sweeps,cooking-time,mfb_cf_fisd,0,RMSE,rmse,lower,0.4633989882463895,-0.4633989882463895,0.4810100390661497,mean,exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0,False,17982 +sweeps,cooking-time,all_four_combined,0,RMSE,rmse,lower,0.463355012524859,-0.463355012524859,0.4809202538385541,mean,exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0,False,17982 +sweeps,cooking-time,all_minus_rla,0,RMSE,rmse,lower,0.4634391174751191,-0.4634391174751191,0.4810216869909678,mean,exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0,False,17982 +sweeps,cooking-time,all_minus_esam,0,RMSE,rmse,lower,0.4633378873945458,-0.4633378873945458,0.48088029743109706,mean,exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0,False,17982 +sweeps,cooking-time,all_minus_mfb,0,RMSE,rmse,lower,0.46319753395446966,-0.46319753395446966,0.48088071575887986,mean,exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0,False,17982 +sweeps,cooking-time,all_minus_cf_fisd,0,RMSE,rmse,lower,0.463067499804912,-0.463067499804912,0.4804862724768644,mean,exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0,False,17982 +sweeps,cooking-time,rla_rank2_noise0.001,0,RMSE,rmse,lower,0.465139054815724,-0.465139054815724,0.48220568612119014,best-head,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-best-head-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank2_noise0.001,0,RMSE,rmse,lower,0.4624384832710735,-0.4624384832710735,0.47998788853107577,greedy-heads,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank2_noise0.0001,0,RMSE,rmse,lower,0.465058894016931,-0.465058894016931,0.48247512338182996,best-head,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-best-head-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank2_noise0.0001,0,RMSE,rmse,lower,0.4624788086134533,-0.4624788086134533,0.4800547545217561,greedy-heads,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank2_noise1e-05,0,RMSE,rmse,lower,0.4650147388029769,-0.4650147388029769,0.4822676099238603,best-head,exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-best-head-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank2_noise1e-05,0,RMSE,rmse,lower,0.46249997680251603,-0.46249997680251603,0.47995778958671514,greedy-heads,exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.001,0,RMSE,rmse,lower,0.4650444270773418,-0.4650444270773418,0.48297671022399175,best-head,exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.001-best-head-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.001,0,RMSE,rmse,lower,0.4625151676835406,-0.4625151676835406,0.479742369480289,greedy-heads,exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.001-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.0025,0,RMSE,rmse,lower,0.46513258351328474,-0.46513258351328474,0.48253251942068065,best-head,exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.0025-best-head-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.0025,0,RMSE,rmse,lower,0.46246732196836127,-0.46246732196836127,0.4800169922135662,greedy-heads,exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.0025-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.005,0,RMSE,rmse,lower,0.4651283707110853,-0.4651283707110853,0.4823068025931435,best-head,exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.005-best-head-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.005,0,RMSE,rmse,lower,0.4625398779765968,-0.4625398779765968,0.4797024392153251,greedy-heads,exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.005-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.7,0,RMSE,rmse,lower,0.46477365365967316,-0.46477365365967316,0.48197477680705497,best-head,exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.7-best-head-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.7,0,RMSE,rmse,lower,0.46249551448708903,-0.46249551448708903,0.47980768580603567,greedy-heads,exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.7-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.8,0,RMSE,rmse,lower,0.4652213638497702,-0.4652213638497702,0.48343941871247514,best-head,exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.8-best-head-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.8,0,RMSE,rmse,lower,0.4624822561623469,-0.4624822561623469,0.4798057758257153,greedy-heads,exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.8-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.9,0,RMSE,rmse,lower,0.465403548325863,-0.465403548325863,0.48311284423774803,best-head,exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.9-best-head-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.9,0,RMSE,rmse,lower,0.4625593198933149,-0.4625593198933149,0.479899744054174,greedy-heads,exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.9-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank4_noise0.001,0,RMSE,rmse,lower,0.4649795846661889,-0.4649795846661889,0.4827695666979872,best-head,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-best-head-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank4_noise0.001,0,RMSE,rmse,lower,0.4624107384246987,-0.4624107384246987,0.479974942646056,greedy-heads,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.01,0,RMSE,rmse,lower,0.46502524927765376,-0.46502524927765376,0.48171681453527604,best-head,exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.01-best-head-evaluation/0,False,23088 +sweeps,cooking-time,esam_rho0.01,0,RMSE,rmse,lower,0.46260330683144324,-0.46260330683144324,0.47994838230926673,greedy-heads,exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_rho0.01-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,cf_fisd_lambda0.05,0,RMSE,rmse,lower,0.4638214410008231,-0.4638214410008231,0.4819659035779245,best-head,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-best-head-evaluation/0,False,17982 +sweeps,cooking-time,cf_fisd_lambda0.05,0,RMSE,rmse,lower,0.4628482268025946,-0.4628482268025946,0.4803119031248436,greedy-heads,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-greedy-heads-evaluation/0,False,17982 +sweeps,cooking-time,rla_rank4_noise0.0001,0,RMSE,rmse,lower,0.4651183431272534,-0.4651183431272534,0.48213882482447107,best-head,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-best-head-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank4_noise0.0001,0,RMSE,rmse,lower,0.4624375326912171,-0.4624375326912171,0.48004674599435365,greedy-heads,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.95,0,RMSE,rmse,lower,0.4649031464912309,-0.4649031464912309,0.4828450127182003,best-head,exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.95-best-head-evaluation/0,False,23088 +sweeps,cooking-time,mfb_keep0.95,0,RMSE,rmse,lower,0.46250350473983365,-0.46250350473983365,0.4799452154615134,greedy-heads,exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_keep0.95-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,cf_fisd_lambda0.1,0,RMSE,rmse,lower,0.46397138529545745,-0.46397138529545745,0.4813020969034895,best-head,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-best-head-evaluation/0,False,17982 +sweeps,cooking-time,cf_fisd_lambda0.1,0,RMSE,rmse,lower,0.4629530240908241,-0.4629530240908241,0.480628211999089,greedy-heads,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-greedy-heads-evaluation/0,False,17982 +sweeps,cooking-time,rla_rank4_noise1e-05,0,RMSE,rmse,lower,0.465206517587419,-0.465206517587419,0.4827162118838719,best-head,exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-best-head-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank4_noise1e-05,0,RMSE,rmse,lower,0.46247985576887213,-0.46247985576887213,0.47992046986293,greedy-heads,exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,cf_fisd_lambda0.2,0,RMSE,rmse,lower,0.4641009735445309,-0.4641009735445309,0.48193517064355434,best-head,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-best-head-evaluation/0,False,15096 +sweeps,cooking-time,cf_fisd_lambda0.2,0,RMSE,rmse,lower,0.463044314080512,-0.463044314080512,0.4804926610439783,greedy-heads,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-greedy-heads-evaluation/0,False,15096 +sweeps,cooking-time,mfb_cf_fisd,0,RMSE,rmse,lower,0.46410829401604725,-0.46410829401604725,0.4815192949196226,best-head,exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_cf_fisd-best-head-evaluation/0,False,17982 +sweeps,cooking-time,mfb_cf_fisd,0,RMSE,rmse,lower,0.46284785656584665,-0.46284785656584665,0.4803782742299631,greedy-heads,exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/mfb_cf_fisd-greedy-heads-evaluation/0,False,17982 +sweeps,cooking-time,rla_rank8_noise0.001,0,RMSE,rmse,lower,0.4648571653594056,-0.4648571653594056,0.4833629710740515,best-head,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-best-head-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank8_noise0.001,0,RMSE,rmse,lower,0.4624392405112558,-0.4624392405112558,0.4799761534270821,greedy-heads,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank8_noise0.0001,0,RMSE,rmse,lower,0.46500539773966587,-0.46500539773966587,0.4826475534492275,best-head,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-best-head-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank8_noise0.0001,0,RMSE,rmse,lower,0.4625085630139665,-0.4625085630139665,0.4800564927891271,greedy-heads,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank8_noise1e-05,0,RMSE,rmse,lower,0.46497304704958375,-0.46497304704958375,0.4824692397862243,best-head,exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-best-head-evaluation/0,False,23088 +sweeps,cooking-time,rla_rank8_noise1e-05,0,RMSE,rmse,lower,0.46244718340270563,-0.46244718340270563,0.4799796305248435,greedy-heads,exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,rla_mfb,0,RMSE,rmse,lower,0.46542866555060514,-0.46542866555060514,0.48315113561982953,best-head,exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_mfb-best-head-evaluation/0,False,23088 +sweeps,cooking-time,rla_mfb,0,RMSE,rmse,lower,0.46255113731378883,-0.46255113731378883,0.4799569823688197,greedy-heads,exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_mfb-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,esam_cf_fisd,0,RMSE,rmse,lower,0.4639245891190366,-0.4639245891190366,0.48198567488759675,best-head,exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_cf_fisd-best-head-evaluation/0,False,17982 +sweeps,cooking-time,esam_cf_fisd,0,RMSE,rmse,lower,0.4629679265241866,-0.4629679265241866,0.4805148498004945,greedy-heads,exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_cf_fisd-greedy-heads-evaluation/0,False,17982 +sweeps,cooking-time,esam_mfb,0,RMSE,rmse,lower,0.46541405002003006,-0.46541405002003006,0.4825725860549557,best-head,exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_mfb-best-head-evaluation/0,False,23088 +sweeps,cooking-time,esam_mfb,0,RMSE,rmse,lower,0.4625995219609569,-0.4625995219609569,0.47992605869184196,greedy-heads,exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/cooking-time/esam_mfb-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,all_minus_rla,0,RMSE,rmse,lower,0.4640838759408798,-0.4640838759408798,0.48156976542107044,best-head,exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_rla-best-head-evaluation/0,False,17982 +sweeps,cooking-time,all_minus_rla,0,RMSE,rmse,lower,0.46292271885090186,-0.46292271885090186,0.48053390557368486,greedy-heads,exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_rla-greedy-heads-evaluation/0,False,17982 +sweeps,cooking-time,rla_cf_fisd,0,RMSE,rmse,lower,0.4638249749501882,-0.4638249749501882,0.4819157837704579,best-head,exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_cf_fisd-best-head-evaluation/0,False,17982 +sweeps,cooking-time,rla_cf_fisd,0,RMSE,rmse,lower,0.4627888241622746,-0.4627888241622746,0.480810230139191,greedy-heads,exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_cf_fisd-greedy-heads-evaluation/0,False,17982 +sweeps,cooking-time,all_minus_esam,0,RMSE,rmse,lower,0.46397896473985684,-0.46397896473985684,0.481064806640852,best-head,exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_esam-best-head-evaluation/0,False,17982 +sweeps,cooking-time,all_minus_esam,0,RMSE,rmse,lower,0.4629167637909319,-0.4629167637909319,0.4803221564231222,greedy-heads,exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_esam-greedy-heads-evaluation/0,False,17982 +sweeps,cooking-time,rla_esam,0,RMSE,rmse,lower,0.46492255368357066,-0.46492255368357066,0.4825495500767423,best-head,exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_esam-best-head-evaluation/0,False,23088 +sweeps,cooking-time,rla_esam,0,RMSE,rmse,lower,0.4624234187671886,-0.4624234187671886,0.4800586966547807,greedy-heads,exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0.toml,exp/integrated/sweeps/cooking-time/rla_esam-greedy-heads-evaluation/0,False,23088 +sweeps,cooking-time,all_minus_cf_fisd,0,RMSE,rmse,lower,0.4649621508176366,-0.4649621508176366,0.48187581714110606,best-head,exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-best-head-evaluation/0,False,17982 +sweeps,cooking-time,all_minus_cf_fisd,0,RMSE,rmse,lower,0.46276401446104365,-0.46276401446104365,0.4800152693247022,greedy-heads,exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-greedy-heads-evaluation/0,False,17982 +sweeps,cooking-time,all_minus_mfb,0,RMSE,rmse,lower,0.46380663024741176,-0.46380663024741176,0.48139068107848576,best-head,exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_mfb-best-head-evaluation/0,False,17982 +sweeps,cooking-time,all_minus_mfb,0,RMSE,rmse,lower,0.4628484521639483,-0.4628484521639483,0.4805722784138239,greedy-heads,exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_minus_mfb-greedy-heads-evaluation/0,False,17982 +sweeps,cooking-time,all_four_combined,0,RMSE,rmse,lower,0.46391938568359065,-0.46391938568359065,0.48142889277254225,best-head,exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_four_combined-best-head-evaluation/0,False,17982 +sweeps,cooking-time,all_four_combined,0,RMSE,rmse,lower,0.4629352081332738,-0.4629352081332738,0.4802514336856013,greedy-heads,exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/cooking-time/all_four_combined-greedy-heads-evaluation/0,False,17982 +sweeps,delivery-eta,rla_rank2_noise0.001,0,RMSE,rmse,lower,0.5556336245387727,-0.5556336245387727,0.5518591323593222,mean,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank2_noise0.0001,0,RMSE,rmse,lower,0.5558827645265865,-0.5558827645265865,0.5504866820118107,mean,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank2_noise1e-05,0,RMSE,rmse,lower,0.5559787764083384,-0.5559787764083384,0.5500968306007196,mean,exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank4_noise0.001,0,RMSE,rmse,lower,0.5564461619356259,-0.5564461619356259,0.5491469465177617,mean,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank4_noise0.0001,0,RMSE,rmse,lower,0.5556926485294449,-0.5556926485294449,0.5502675684403805,mean,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank4_noise1e-05,0,RMSE,rmse,lower,0.5560204514474498,-0.5560204514474498,0.5515690033180755,mean,exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank8_noise0.001,0,RMSE,rmse,lower,0.5561609443354546,-0.5561609443354546,0.5482846212731874,mean,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0,False,1638 +sweeps,delivery-eta,rla_rank8_noise0.0001,0,RMSE,rmse,lower,0.5572424061712593,-0.5572424061712593,0.546917368746242,mean,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank8_noise1e-05,0,RMSE,rmse,lower,0.5558005167077797,-0.5558005167077797,0.5500678724978622,mean,exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0,False,1365 +sweeps,delivery-eta,esam_rho0.001,0,RMSE,rmse,lower,0.5561053998926777,-0.5561053998926777,0.5493164333767354,mean,exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0,False,1365 +sweeps,delivery-eta,esam_rho0.0025,0,RMSE,rmse,lower,0.5559746489354217,-0.5559746489354217,0.5535077457230605,mean,exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0,False,2184 +sweeps,delivery-eta,esam_rho0.005,0,RMSE,rmse,lower,0.5556360918188752,-0.5556360918188752,0.5494649592968895,mean,exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0,False,2184 +sweeps,delivery-eta,esam_rho0.01,0,RMSE,rmse,lower,0.555695893192201,-0.555695893192201,0.5503082950221266,mean,exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0,False,1365 +sweeps,delivery-eta,mfb_keep0.7,0,RMSE,rmse,lower,0.5576335149669441,-0.5576335149669441,0.5507854163911843,mean,exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0,False,2730 +sweeps,delivery-eta,mfb_keep0.8,0,RMSE,rmse,lower,0.5567305910455574,-0.5567305910455574,0.5489749915571137,mean,exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0,False,2730 +sweeps,delivery-eta,mfb_keep0.9,0,RMSE,rmse,lower,0.5560026829864069,-0.5560026829864069,0.5501201530906976,mean,exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0,False,2184 +sweeps,delivery-eta,mfb_keep0.95,0,RMSE,rmse,lower,0.5557158166151223,-0.5557158166151223,0.5496765041498511,mean,exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0,False,2184 +sweeps,delivery-eta,cf_fisd_lambda0.05,0,RMSE,rmse,lower,0.5562445854609275,-0.5562445854609275,0.5504166498793033,mean,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0,False,2184 +sweeps,delivery-eta,cf_fisd_lambda0.1,0,RMSE,rmse,lower,0.5562691235201772,-0.5562691235201772,0.5515996385557942,mean,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0,False,3003 +sweeps,delivery-eta,cf_fisd_lambda0.2,0,RMSE,rmse,lower,0.5565664676993906,-0.5565664676993906,0.5475279860550825,mean,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0,False,1638 +sweeps,delivery-eta,rla_esam,0,RMSE,rmse,lower,0.5559417888696784,-0.5559417888696784,0.5541331785404355,mean,exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0,False,1365 +sweeps,delivery-eta,rla_mfb,0,RMSE,rmse,lower,0.5559017430643394,-0.5559017430643394,0.5492186599232151,mean,exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0,False,1365 +sweeps,delivery-eta,rla_cf_fisd,0,RMSE,rmse,lower,0.556067616855386,-0.556067616855386,0.5512732344334902,mean,exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0,False,1638 +sweeps,delivery-eta,esam_mfb,0,RMSE,rmse,lower,0.5560703233852616,-0.5560703233852616,0.5508616503748457,mean,exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0,False,2730 +sweeps,delivery-eta,esam_cf_fisd,0,RMSE,rmse,lower,0.5562165100217151,-0.5562165100217151,0.5500920359593422,mean,exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0,False,1638 +sweeps,delivery-eta,mfb_cf_fisd,0,RMSE,rmse,lower,0.55593889408891,-0.55593889408891,0.5537058512136184,mean,exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0,False,1638 +sweeps,delivery-eta,all_four_combined,0,RMSE,rmse,lower,0.5560789252392828,-0.5560789252392828,0.5493995976501785,mean,exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0,False,1638 +sweeps,delivery-eta,all_minus_rla,0,RMSE,rmse,lower,0.5560596579733861,-0.5560596579733861,0.5513667248174312,mean,exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0,False,1638 +sweeps,delivery-eta,all_minus_esam,0,RMSE,rmse,lower,0.556102345189303,-0.556102345189303,0.5505710768952337,mean,exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0,False,1365 +sweeps,delivery-eta,all_minus_mfb,0,RMSE,rmse,lower,0.5571737580629273,-0.5571737580629273,0.5527981832493051,mean,exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0,False,2184 +sweeps,delivery-eta,all_minus_cf_fisd,0,RMSE,rmse,lower,0.5563682291205534,-0.5563682291205534,0.5506428213190716,mean,exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0,False,2184 +sweeps,delivery-eta,rla_rank2_noise0.001,0,RMSE,rmse,lower,0.5574634025946379,-0.5574634025946379,0.5547066537478277,best-head,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank2_noise0.001,0,RMSE,rmse,lower,0.5552720495549479,-0.5552720495549479,0.5524877260752137,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank2_noise0.0001,0,RMSE,rmse,lower,0.5574308976082757,-0.5574308976082757,0.5550464489547554,best-head,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank2_noise0.0001,0,RMSE,rmse,lower,0.5553755991930566,-0.5553755991930566,0.5516112816611277,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank2_noise1e-05,0,RMSE,rmse,lower,0.5574473374558679,-0.5574473374558679,0.5570882235759158,best-head,exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank2_noise1e-05,0,RMSE,rmse,lower,0.5553479091365543,-0.5553479091365543,0.5511482044438852,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank4_noise0.001,0,RMSE,rmse,lower,0.5571872637155045,-0.5571872637155045,0.5515561165588831,best-head,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank4_noise0.001,0,RMSE,rmse,lower,0.5552950473462183,-0.5552950473462183,0.5512393912767233,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank4_noise0.0001,0,RMSE,rmse,lower,0.5573103773832353,-0.5573103773832353,0.5532468973961396,best-head,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank4_noise0.0001,0,RMSE,rmse,lower,0.5553051102575359,-0.5553051102575359,0.5508335169889497,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,esam_rho0.001,0,RMSE,rmse,lower,0.5577754983463842,-0.5577754983463842,0.5539871683993082,best-head,exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.001-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,esam_rho0.001,0,RMSE,rmse,lower,0.5555279503030305,-0.5555279503030305,0.5511287648017675,greedy-heads,exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.001-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank4_noise1e-05,0,RMSE,rmse,lower,0.5569447272303437,-0.5569447272303437,0.5529624818746817,best-head,exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank4_noise1e-05,0,RMSE,rmse,lower,0.5553112552523684,-0.5553112552523684,0.552273210826067,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,esam_rho0.0025,0,RMSE,rmse,lower,0.5594097126680496,-0.5594097126680496,0.5568830263477744,best-head,exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.0025-best-head-evaluation/0,False,2184 +sweeps,delivery-eta,esam_rho0.0025,0,RMSE,rmse,lower,0.5555843034564808,-0.5555843034564808,0.5528859176747987,greedy-heads,exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.0025-greedy-heads-evaluation/0,False,2184 +sweeps,delivery-eta,esam_rho0.005,0,RMSE,rmse,lower,0.5582863288518123,-0.5582863288518123,0.5560390501031047,best-head,exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.005-best-head-evaluation/0,False,2184 +sweeps,delivery-eta,esam_rho0.005,0,RMSE,rmse,lower,0.5551513290724005,-0.5551513290724005,0.5507488105906536,greedy-heads,exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.005-greedy-heads-evaluation/0,False,2184 +sweeps,delivery-eta,esam_rho0.01,0,RMSE,rmse,lower,0.5577866919608585,-0.5577866919608585,0.5538906498744642,best-head,exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.01-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,esam_rho0.01,0,RMSE,rmse,lower,0.5553495995574796,-0.5553495995574796,0.5513616169057922,greedy-heads,exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_rho0.01-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank8_noise0.0001,0,RMSE,rmse,lower,0.5573390661266857,-0.5573390661266857,0.5535791095517189,best-head,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank8_noise0.0001,0,RMSE,rmse,lower,0.5558846409645507,-0.5558846409645507,0.5506559188769905,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank8_noise0.001,0,RMSE,rmse,lower,0.5576981787979653,-0.5576981787979653,0.5514806270620619,best-head,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-best-head-evaluation/0,False,1638 +sweeps,delivery-eta,rla_rank8_noise0.001,0,RMSE,rmse,lower,0.5556463362777673,-0.5556463362777673,0.5491140035096563,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-greedy-heads-evaluation/0,False,1638 +sweeps,delivery-eta,rla_rank8_noise1e-05,0,RMSE,rmse,lower,0.5574517747950514,-0.5574517747950514,0.5534907850292722,best-head,exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,rla_rank8_noise1e-05,0,RMSE,rmse,lower,0.5553793554897237,-0.5553793554897237,0.5521969288494308,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,cf_fisd_lambda0.2,0,RMSE,rmse,lower,0.5568247172798395,-0.5568247172798395,0.5526535729970853,best-head,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-best-head-evaluation/0,False,1638 +sweeps,delivery-eta,cf_fisd_lambda0.2,0,RMSE,rmse,lower,0.5558508909198594,-0.5558508909198594,0.5511616144232316,greedy-heads,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-greedy-heads-evaluation/0,False,1638 +sweeps,delivery-eta,cf_fisd_lambda0.05,0,RMSE,rmse,lower,0.557661572508718,-0.557661572508718,0.553144673358601,best-head,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-best-head-evaluation/0,False,2184 +sweeps,delivery-eta,cf_fisd_lambda0.05,0,RMSE,rmse,lower,0.5556691040453957,-0.5556691040453957,0.5521822756750591,greedy-heads,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-greedy-heads-evaluation/0,False,2184 +sweeps,delivery-eta,mfb_cf_fisd,0,RMSE,rmse,lower,0.556619663847731,-0.556619663847731,0.55593546321811,best-head,exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-best-head-evaluation/0,False,1638 +sweeps,delivery-eta,mfb_cf_fisd,0,RMSE,rmse,lower,0.5555428639260286,-0.5555428639260286,0.5513865344186518,greedy-heads,exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-greedy-heads-evaluation/0,False,1638 +sweeps,delivery-eta,mfb_keep0.95,0,RMSE,rmse,lower,0.5584568843028204,-0.5584568843028204,0.5549022634897627,best-head,exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.95-best-head-evaluation/0,False,2184 +sweeps,delivery-eta,mfb_keep0.9,0,RMSE,rmse,lower,0.5590040394051904,-0.5590040394051904,0.5557457406296488,best-head,exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.9-best-head-evaluation/0,False,2184 +sweeps,delivery-eta,mfb_keep0.95,0,RMSE,rmse,lower,0.5552807442811304,-0.5552807442811304,0.5512821544083845,greedy-heads,exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.95-greedy-heads-evaluation/0,False,2184 +sweeps,delivery-eta,mfb_keep0.9,0,RMSE,rmse,lower,0.5555016895512279,-0.5555016895512279,0.5503428181880856,greedy-heads,exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.9-greedy-heads-evaluation/0,False,2184 +sweeps,delivery-eta,cf_fisd_lambda0.1,0,RMSE,rmse,lower,0.5580564462352832,-0.5580564462352832,0.5528824139554394,best-head,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-best-head-evaluation/0,False,3003 +sweeps,delivery-eta,cf_fisd_lambda0.1,0,RMSE,rmse,lower,0.5559326756199947,-0.5559326756199947,0.5522105562224877,greedy-heads,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-greedy-heads-evaluation/0,False,3003 +sweeps,delivery-eta,mfb_keep0.7,0,RMSE,rmse,lower,0.5595885535124743,-0.5595885535124743,0.5534224254724212,best-head,exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.7-best-head-evaluation/0,False,2730 +sweeps,delivery-eta,mfb_keep0.7,0,RMSE,rmse,lower,0.5551008108444078,-0.5551008108444078,0.5501919833980724,greedy-heads,exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.7-greedy-heads-evaluation/0,False,2730 +sweeps,delivery-eta,mfb_keep0.8,0,RMSE,rmse,lower,0.5594878077640214,-0.5594878077640214,0.5558040556519808,best-head,exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.8-best-head-evaluation/0,False,2730 +sweeps,delivery-eta,mfb_keep0.8,0,RMSE,rmse,lower,0.5554990875501031,-0.5554990875501031,0.5493161621093208,greedy-heads,exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/mfb_keep0.8-greedy-heads-evaluation/0,False,2730 +sweeps,delivery-eta,esam_cf_fisd,0,RMSE,rmse,lower,0.5568538056781229,-0.5568538056781229,0.5520497317854985,best-head,exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_cf_fisd-best-head-evaluation/0,False,1638 +sweeps,delivery-eta,esam_cf_fisd,0,RMSE,rmse,lower,0.5558497381815869,-0.5558497381815869,0.5508600814353861,greedy-heads,exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_cf_fisd-greedy-heads-evaluation/0,False,1638 +sweeps,delivery-eta,all_minus_rla,0,RMSE,rmse,lower,0.5572234465535005,-0.5572234465535005,0.5525491087220346,best-head,exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_rla-best-head-evaluation/0,False,1638 +sweeps,delivery-eta,all_minus_rla,0,RMSE,rmse,lower,0.5556365745463094,-0.5556365745463094,0.5515955863733182,greedy-heads,exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_rla-greedy-heads-evaluation/0,False,1638 +sweeps,delivery-eta,esam_mfb,0,RMSE,rmse,lower,0.5602888251013873,-0.5602888251013873,0.5556835311938418,best-head,exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_mfb-best-head-evaluation/0,False,2730 +sweeps,delivery-eta,esam_mfb,0,RMSE,rmse,lower,0.5556538720026133,-0.5556538720026133,0.5520326723090689,greedy-heads,exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/esam_mfb-greedy-heads-evaluation/0,False,2730 +sweeps,delivery-eta,rla_cf_fisd,0,RMSE,rmse,lower,0.5567133805762534,-0.5567133805762534,0.5518657747493417,best-head,exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_cf_fisd-best-head-evaluation/0,False,1638 +sweeps,delivery-eta,rla_cf_fisd,0,RMSE,rmse,lower,0.55586820851409,-0.55586820851409,0.5514142072129953,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_cf_fisd-greedy-heads-evaluation/0,False,1638 +sweeps,delivery-eta,rla_mfb,0,RMSE,rmse,lower,0.5579516581852805,-0.5579516581852805,0.5533480523154869,best-head,exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_mfb-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,rla_mfb,0,RMSE,rmse,lower,0.5554524372350571,-0.5554524372350571,0.5505269863779684,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_mfb-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,all_minus_esam,0,RMSE,rmse,lower,0.5574872455078781,-0.5574872455078781,0.5557374285650164,best-head,exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_esam-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,all_minus_esam,0,RMSE,rmse,lower,0.5556806886998003,-0.5556806886998003,0.5513624006625033,greedy-heads,exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_esam-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,rla_esam,0,RMSE,rmse,lower,0.5572907783524989,-0.5572907783524989,0.5545303498210034,best-head,exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_esam-best-head-evaluation/0,False,1365 +sweeps,delivery-eta,rla_esam,0,RMSE,rmse,lower,0.5552988041877751,-0.5552988041877751,0.5525187688548042,greedy-heads,exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/rla_esam-greedy-heads-evaluation/0,False,1365 +sweeps,delivery-eta,all_four_combined,0,RMSE,rmse,lower,0.557107642501001,-0.557107642501001,0.5520572896128711,best-head,exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_four_combined-best-head-evaluation/0,False,1638 +sweeps,delivery-eta,all_four_combined,0,RMSE,rmse,lower,0.5558210261849236,-0.5558210261849236,0.551614874495521,greedy-heads,exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_four_combined-greedy-heads-evaluation/0,False,1638 +sweeps,delivery-eta,all_minus_mfb,0,RMSE,rmse,lower,0.5591485795972553,-0.5591485795972553,0.5570234889363207,best-head,exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_mfb-best-head-evaluation/0,False,2184 +sweeps,delivery-eta,all_minus_mfb,0,RMSE,rmse,lower,0.5566172544693042,-0.5566172544693042,0.5524610510858833,greedy-heads,exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_mfb-greedy-heads-evaluation/0,False,2184 +sweeps,delivery-eta,all_minus_cf_fisd,0,RMSE,rmse,lower,0.5589936165675472,-0.5589936165675472,0.5554125439499674,best-head,exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-best-head-evaluation/0,False,2184 +sweeps,delivery-eta,all_minus_cf_fisd,0,RMSE,rmse,lower,0.5557239681192615,-0.5557239681192615,0.5525121073453494,greedy-heads,exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0.toml,exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-greedy-heads-evaluation/0,False,2184 +final,sberbank-housing,baseline_plr,0,RMSE,rmse,lower,0.23452539387100468,-0.23452539387100468,0.23846659721529556,mean,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0,False,814 +final,sberbank-housing,baseline_plr,2,RMSE,rmse,lower,0.23581900020763122,-0.23581900020763122,0.23293721743587778,mean,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2,False,666 +final,sberbank-housing,baseline_plr,1,RMSE,rmse,lower,0.23623006536000674,-0.23623006536000674,0.2318497676950212,mean,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1,False,962 +final,sberbank-housing,best_rla_only,0,RMSE,rmse,lower,0.2354682854519358,-0.2354682854519358,0.24014251007009368,mean,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0,False,814 +final,sberbank-housing,best_rla_only,1,RMSE,rmse,lower,0.23689852205208367,-0.23689852205208367,0.23132638249744272,mean,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1,False,814 +final,sberbank-housing,best_rla_only,2,RMSE,rmse,lower,0.2359145620703149,-0.2359145620703149,0.2332252993496603,mean,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2,False,666 +final,sberbank-housing,best_esam_only,0,RMSE,rmse,lower,0.23474623366768377,-0.23474623366768377,0.23936563514587833,mean,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0,False,814 +final,sberbank-housing,best_esam_only,2,RMSE,rmse,lower,0.2356560348188032,-0.2356560348188032,0.23118945759761478,mean,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2,False,814 +final,sberbank-housing,best_esam_only,1,RMSE,rmse,lower,0.23723213436744667,-0.23723213436744667,0.23099161094856513,mean,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1,False,814 +final,sberbank-housing,best_mfb_only,0,RMSE,rmse,lower,0.2341279953511937,-0.2341279953511937,0.23402994505001048,mean,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0,False,888 +final,sberbank-housing,best_mfb_only,1,RMSE,rmse,lower,0.23621923128975542,-0.23621923128975542,0.23276193568912745,mean,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1,False,592 +final,sberbank-housing,best_mfb_only,2,RMSE,rmse,lower,0.23605616345434166,-0.23605616345434166,0.23451195530114213,mean,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2,False,1110 +final,sberbank-housing,best_cf_fisd_only,0,RMSE,rmse,lower,0.23523413233394516,-0.23523413233394516,0.235149605552611,mean,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0,False,666 +final,sberbank-housing,best_cf_fisd_only,1,RMSE,rmse,lower,0.23639255622691643,-0.23639255622691643,0.2324256095265638,mean,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1,False,814 +final,sberbank-housing,best_cf_fisd_only,2,RMSE,rmse,lower,0.2359054032023901,-0.2359054032023901,0.23245970634502372,mean,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2,False,518 +final,sberbank-housing,best_combined,0,RMSE,rmse,lower,0.23508488120210985,-0.23508488120210985,0.23568789402876444,mean,exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_combined-evaluation/0,False,888 +final,sberbank-housing,best_combined,1,RMSE,rmse,lower,0.23530641494725335,-0.23530641494725335,0.23169352906704774,mean,exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_combined-evaluation/1,False,592 +final,sberbank-housing,best_combined,2,RMSE,rmse,lower,0.2356153727014129,-0.2356153727014129,0.2360324823167294,mean,exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_combined-evaluation/2,False,1036 +final,sberbank-housing,baseline_plr,0,RMSE,rmse,lower,0.23955454428788905,-0.23955454428788905,0.26588264760298186,best-head,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml,exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/0,False,814 +final,sberbank-housing,baseline_plr,2,RMSE,rmse,lower,0.23944956044068919,-0.23944956044068919,0.24308412455882378,best-head,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml,exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/2,False,666 +final,sberbank-housing,baseline_plr,1,RMSE,rmse,lower,0.24150787640249508,-0.24150787640249508,0.2416341058127328,best-head,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml,exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/1,False,962 +final,sberbank-housing,baseline_plr,0,RMSE,rmse,lower,0.23257941028161666,-0.23257941028161666,0.23549883338311248,greedy-heads,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0,False,814 +final,sberbank-housing,baseline_plr,2,RMSE,rmse,lower,0.23449523543792233,-0.23449523543792233,0.23591672540617004,greedy-heads,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,False,666 +final,sberbank-housing,baseline_plr,1,RMSE,rmse,lower,0.2341956008327552,-0.2341956008327552,0.23089961816224172,greedy-heads,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1,False,962 +final,sberbank-housing,best_rla_only,0,RMSE,rmse,lower,0.2370483756746748,-0.2370483756746748,0.23920040609688478,best-head,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/0,False,814 +final,sberbank-housing,best_rla_only,2,RMSE,rmse,lower,0.2381416834471136,-0.2381416834471136,0.2498549025398126,best-head,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/2,False,666 +final,sberbank-housing,best_rla_only,1,RMSE,rmse,lower,0.23898563084143007,-0.23898563084143007,0.23816985512587505,best-head,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/1,False,814 +final,sberbank-housing,best_rla_only,0,RMSE,rmse,lower,0.2328613038082034,-0.2328613038082034,0.23498425768312764,greedy-heads,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0,False,814 +final,sberbank-housing,best_rla_only,2,RMSE,rmse,lower,0.23410085698681354,-0.23410085698681354,0.23135944978887066,greedy-heads,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2,False,666 +final,sberbank-housing,best_rla_only,1,RMSE,rmse,lower,0.23460022912632159,-0.23460022912632159,0.23076016380764466,greedy-heads,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1,False,814 +final,sberbank-housing,best_esam_only,0,RMSE,rmse,lower,0.2377136850270048,-0.2377136850270048,0.262345673404661,best-head,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/0,False,814 +final,sberbank-housing,best_esam_only,2,RMSE,rmse,lower,0.24089231567126032,-0.24089231567126032,0.24860647860679874,best-head,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/2,False,814 +final,sberbank-housing,best_esam_only,1,RMSE,rmse,lower,0.2385787982165438,-0.2385787982165438,0.24822352719647534,best-head,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/1,False,814 +final,sberbank-housing,best_esam_only,0,RMSE,rmse,lower,0.23295802298295287,-0.23295802298295287,0.23903366020873654,greedy-heads,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0,False,814 +final,sberbank-housing,best_esam_only,2,RMSE,rmse,lower,0.23456056719125185,-0.23456056719125185,0.23191054380220286,greedy-heads,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2,False,814 +final,sberbank-housing,best_esam_only,1,RMSE,rmse,lower,0.23392588182013857,-0.23392588182013857,0.23227402448302542,greedy-heads,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1,False,814 +final,sberbank-housing,best_mfb_only,0,RMSE,rmse,lower,0.2380346366558266,-0.2380346366558266,0.2579307429494123,best-head,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/0,False,888 +final,sberbank-housing,best_mfb_only,1,RMSE,rmse,lower,0.2385396806698102,-0.2385396806698102,0.2407010051283554,best-head,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/1,False,592 +final,sberbank-housing,best_mfb_only,2,RMSE,rmse,lower,0.24181054374010003,-0.24181054374010003,0.24863506025450496,best-head,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/2,False,1110 +final,sberbank-housing,best_mfb_only,0,RMSE,rmse,lower,0.23237596587359466,-0.23237596587359466,0.23064696162155762,greedy-heads,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0,False,888 +final,sberbank-housing,best_mfb_only,1,RMSE,rmse,lower,0.23377133352416926,-0.23377133352416926,0.23643993065905872,greedy-heads,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1,False,592 +final,sberbank-housing,best_mfb_only,2,RMSE,rmse,lower,0.23350988305065445,-0.23350988305065445,0.23821200474658916,greedy-heads,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2,False,1110 +final,sberbank-housing,best_cf_fisd_only,0,RMSE,rmse,lower,0.2388197411165918,-0.2388197411165918,0.2462523116085554,best-head,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/0,False,666 +final,sberbank-housing,best_cf_fisd_only,2,RMSE,rmse,lower,0.2376592052308581,-0.2376592052308581,0.23311093711112285,best-head,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/2,False,518 +final,sberbank-housing,best_cf_fisd_only,1,RMSE,rmse,lower,0.24061445857155628,-0.24061445857155628,0.23970045288478414,best-head,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/1,False,814 +final,sberbank-housing,best_cf_fisd_only,0,RMSE,rmse,lower,0.23327114512588593,-0.23327114512588593,0.23367141178994547,greedy-heads,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0,False,666 +final,sberbank-housing,best_cf_fisd_only,2,RMSE,rmse,lower,0.23432768503071347,-0.23432768503071347,0.2325014731401514,greedy-heads,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2,False,518 +final,sberbank-housing,best_cf_fisd_only,1,RMSE,rmse,lower,0.23463880474397839,-0.23463880474397839,0.2331063026442073,greedy-heads,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1,False,814 +final,sberbank-housing,best_combined,0,RMSE,rmse,lower,0.23765498863853576,-0.23765498863853576,0.2536206620016002,best-head,exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0,False,888 +final,sberbank-housing,best_combined,1,RMSE,rmse,lower,0.2374959967300924,-0.2374959967300924,0.24912023719870632,best-head,exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1,False,592 +final,sberbank-housing,best_combined,2,RMSE,rmse,lower,0.23866731604984137,-0.23866731604984137,0.2357692652387237,best-head,exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2,False,1036 +final,sberbank-housing,best_combined,0,RMSE,rmse,lower,0.23320900639668657,-0.23320900639668657,0.23362227205150313,greedy-heads,exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0,False,888 +final,sberbank-housing,best_combined,1,RMSE,rmse,lower,0.2332021853813015,-0.2332021853813015,0.2324977398321572,greedy-heads,exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1,False,592 +final,sberbank-housing,best_combined,2,RMSE,rmse,lower,0.23346459476611428,-0.23346459476611428,0.23881630936614148,greedy-heads,exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2,False,1036 +final,ecom-offers,baseline_plr,0,AUROC,roc-auc,higher,0.6227054517511603,0.6227054517511603,0.591151981311678,mean,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0,False,321 +final,ecom-offers,baseline_plr,2,AUROC,roc-auc,higher,0.6212503263524413,0.6212503263524413,0.5893011087345269,mean,exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml,exp/integrated/final/ecom-offers/baseline_plr-evaluation/2,False,321 +final,ecom-offers,baseline_plr,1,AUROC,roc-auc,higher,0.6203331748306973,0.6203331748306973,0.5897210540664094,mean,exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml,exp/integrated/final/ecom-offers/baseline_plr-evaluation/1,False,428 +final,ecom-offers,best_rla_only,0,AUROC,roc-auc,higher,0.6227001707049361,0.6227001707049361,0.5911765632808944,mean,exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_rla_only-evaluation/0,False,214 +final,ecom-offers,best_rla_only,1,AUROC,roc-auc,higher,0.6203498803314189,0.6203498803314189,0.5897023859034299,mean,exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_rla_only-evaluation/1,False,428 +final,ecom-offers,best_rla_only,2,AUROC,roc-auc,higher,0.6211526583224122,0.6211526583224122,0.5892307660549951,mean,exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_rla_only-evaluation/2,False,321 +final,ecom-offers,best_esam_only,0,AUROC,roc-auc,higher,0.6227874903903754,0.6227874903903754,0.5911993816464689,mean,exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_esam_only-evaluation/0,False,321 +final,ecom-offers,best_esam_only,1,AUROC,roc-auc,higher,0.6204141557865444,0.6204141557865444,0.5897760078319128,mean,exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_esam_only-evaluation/1,False,428 +final,ecom-offers,best_esam_only,2,AUROC,roc-auc,higher,0.6211550537725488,0.6211550537725488,0.5892400551940371,mean,exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_esam_only-evaluation/2,False,321 +final,ecom-offers,best_mfb_only,0,AUROC,roc-auc,higher,0.6291199291618959,0.6291199291618959,0.5961977102213372,mean,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0,False,428 +final,ecom-offers,best_mfb_only,1,AUROC,roc-auc,higher,0.6253906113125361,0.6253906113125361,0.5930931510151916,mean,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1,False,428 +final,ecom-offers,best_mfb_only,2,AUROC,roc-auc,higher,0.6220034632459894,0.6220034632459894,0.5900390978219552,mean,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2,False,321 +final,ecom-offers,best_cf_fisd_only,0,AUROC,roc-auc,higher,0.6228186386127663,0.6228186386127663,0.5911736900637132,mean,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0,False,321 +final,ecom-offers,best_cf_fisd_only,1,AUROC,roc-auc,higher,0.6202479078617598,0.6202479078617598,0.5895844228260869,mean,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1,False,428 +final,ecom-offers,best_cf_fisd_only,2,AUROC,roc-auc,higher,0.6211436845976699,0.6211436845976699,0.5891830464738508,mean,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2,False,321 +final,ecom-offers,best_combined,0,AUROC,roc-auc,higher,0.6250550271749447,0.6250550271749447,0.592670158895295,mean,exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml,exp/integrated/final/ecom-offers/best_combined-evaluation/0,False,321 +final,ecom-offers,best_combined,1,AUROC,roc-auc,higher,0.6213201555665758,0.6213201555665758,0.5900624771932574,mean,exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml,exp/integrated/final/ecom-offers/best_combined-evaluation/1,False,428 +final,ecom-offers,best_combined,2,AUROC,roc-auc,higher,0.6208476032754839,0.6208476032754839,0.5890089208341323,mean,exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml,exp/integrated/final/ecom-offers/best_combined-evaluation/2,False,321 +final,ecom-offers,baseline_plr,0,AUROC,roc-auc,higher,0.6404461738890127,0.6404461738890127,0.6034781946000056,best-head,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml,exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0,False,321 +final,ecom-offers,baseline_plr,2,AUROC,roc-auc,higher,0.6342271136151092,0.6342271136151092,0.5992387053088735,best-head,exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml,exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2,False,321 +final,ecom-offers,baseline_plr,1,AUROC,roc-auc,higher,0.6313621036574809,0.6313621036574809,0.5963646574673217,best-head,exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml,exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1,False,428 +final,ecom-offers,baseline_plr,0,AUROC,roc-auc,higher,0.6413971749638401,0.6413971749638401,0.6040305589800323,greedy-heads,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml,exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0,False,321 +final,ecom-offers,baseline_plr,2,AUROC,roc-auc,higher,0.6356383175142475,0.6356383175142475,0.5998927728985708,greedy-heads,exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml,exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2,False,321 +final,ecom-offers,baseline_plr,1,AUROC,roc-auc,higher,0.6327357100453991,0.6327357100453991,0.5971884680332056,greedy-heads,exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml,exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1,False,428 +final,ecom-offers,best_rla_only,0,AUROC,roc-auc,higher,0.6396541901648656,0.6396541901648656,0.6015418321886687,best-head,exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0,False,214 +final,ecom-offers,best_rla_only,2,AUROC,roc-auc,higher,0.633681175787757,0.633681175787757,0.597700306723218,best-head,exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2,False,321 +final,ecom-offers,best_rla_only,1,AUROC,roc-auc,higher,0.6311484553024557,0.6311484553024557,0.5960986415689524,best-head,exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1,False,428 +final,ecom-offers,best_rla_only,0,AUROC,roc-auc,higher,0.6396541901648656,0.6396541901648656,0.6015418321886687,greedy-heads,exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/0,False,214 +final,ecom-offers,best_rla_only,2,AUROC,roc-auc,higher,0.6351641473729864,0.6351641473729864,0.5998596611627065,greedy-heads,exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/2,False,321 +final,ecom-offers,best_rla_only,1,AUROC,roc-auc,higher,0.6323603724914644,0.6323603724914644,0.5968416363152236,greedy-heads,exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/1,False,428 +final,ecom-offers,best_esam_only,0,AUROC,roc-auc,higher,0.6410241296714208,0.6410241296714208,0.6024966050474919,best-head,exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/0,False,321 +final,ecom-offers,best_esam_only,1,AUROC,roc-auc,higher,0.6315182980623079,0.6315182980623079,0.5962485571769538,best-head,exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/1,False,428 +final,ecom-offers,best_esam_only,2,AUROC,roc-auc,higher,0.6341914987995403,0.6341914987995403,0.599267480873393,best-head,exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/2,False,321 +final,ecom-offers,best_esam_only,0,AUROC,roc-auc,higher,0.641845068859757,0.641845068859757,0.6041622806453656,greedy-heads,exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0,False,321 +final,ecom-offers,best_esam_only,2,AUROC,roc-auc,higher,0.6354981541987959,0.6354981541987959,0.5998512677731419,greedy-heads,exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2,False,321 +final,ecom-offers,best_esam_only,1,AUROC,roc-auc,higher,0.6326258289049813,0.6326258289049813,0.5972452690882881,greedy-heads,exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1,False,428 +final,ecom-offers,best_mfb_only,0,AUROC,roc-auc,higher,0.6547032702847417,0.6547032702847417,0.6118564029158264,best-head,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0,False,428 +final,ecom-offers,best_mfb_only,2,AUROC,roc-auc,higher,0.6546592087434608,0.6546592087434608,0.6088559366177836,best-head,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2,False,321 +final,ecom-offers,best_mfb_only,1,AUROC,roc-auc,higher,0.6525683493050402,0.6525683493050402,0.6078370975247275,best-head,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1,False,428 +final,ecom-offers,best_mfb_only,0,AUROC,roc-auc,higher,0.6547032702847417,0.6547032702847417,0.6118564029158264,greedy-heads,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/0,False,428 +final,ecom-offers,best_mfb_only,2,AUROC,roc-auc,higher,0.6546592087434608,0.6546592087434608,0.6088559366177836,greedy-heads,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/2,False,321 +final,ecom-offers,best_mfb_only,1,AUROC,roc-auc,higher,0.6527163070467052,0.6527163070467052,0.6065300223672054,greedy-heads,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/1,False,428 +final,ecom-offers,best_cf_fisd_only,0,AUROC,roc-auc,higher,0.6410202011331968,0.6410202011331968,0.6024701571918213,best-head,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/0,False,321 +final,ecom-offers,best_cf_fisd_only,2,AUROC,roc-auc,higher,0.6333770862914994,0.6333770862914994,0.5972389647476771,best-head,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/2,False,321 +final,ecom-offers,best_cf_fisd_only,1,AUROC,roc-auc,higher,0.6322301668779651,0.6322301668779651,0.5946009422441442,best-head,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/1,False,428 +final,ecom-offers,best_cf_fisd_only,0,AUROC,roc-auc,higher,0.6419149054445072,0.6419149054445072,0.6041224275423289,greedy-heads,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0,False,321 +final,ecom-offers,best_cf_fisd_only,2,AUROC,roc-auc,higher,0.6340638176219543,0.6340638176219543,0.598916340879379,greedy-heads,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2,False,321 +final,ecom-offers,best_cf_fisd_only,1,AUROC,roc-auc,higher,0.6333421643138163,0.6333421643138163,0.5975759215250558,greedy-heads,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1,False,428 +final,ecom-offers,best_combined,0,AUROC,roc-auc,higher,0.6477890245840551,0.6477890245840551,0.6045088303107439,best-head,exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml,exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/0,False,321 +final,ecom-offers,best_combined,2,AUROC,roc-auc,higher,0.6430199007953441,0.6430199007953441,0.5970664756329549,best-head,exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml,exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/2,False,321 +final,ecom-offers,best_combined,1,AUROC,roc-auc,higher,0.6512227217747169,0.6512227217747169,0.6075381341628896,best-head,exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml,exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/1,False,428 +final,ecom-offers,best_combined,0,AUROC,roc-auc,higher,0.6506783465179156,0.6506783465179156,0.6065618416202598,greedy-heads,exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml,exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0,False,321 +final,ecom-offers,best_combined,1,AUROC,roc-auc,higher,0.6512227217747169,0.6512227217747169,0.6075381341628896,greedy-heads,exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml,exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1,False,428 +final,ecom-offers,best_combined,2,AUROC,roc-auc,higher,0.6436592869751769,0.6436592869751769,0.6015328994897303,greedy-heads,exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml,exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2,False,321 +final,homesite-insurance,baseline_plr,0,AUROC,roc-auc,higher,0.9583121450834431,0.9583121450834431,0.9626260391126089,mean,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0,False,1980 +final,homesite-insurance,baseline_plr,1,AUROC,roc-auc,higher,0.9586997364143323,0.9586997364143323,0.9622916084556132,mean,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1,False,1760 +final,homesite-insurance,baseline_plr,2,AUROC,roc-auc,higher,0.9582475934195426,0.9582475934195426,0.9625649084455907,mean,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2,False,1980 +final,homesite-insurance,best_rla_only,0,AUROC,roc-auc,higher,0.9581626616470993,0.9581626616470993,0.9620964494052155,mean,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0,False,1760 +final,homesite-insurance,best_rla_only,1,AUROC,roc-auc,higher,0.9583407546059616,0.9583407546059616,0.9620763221935056,mean,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1,False,1760 +final,homesite-insurance,best_rla_only,2,AUROC,roc-auc,higher,0.9581674973070209,0.9581674973070209,0.9627480104988175,mean,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2,False,1980 +final,homesite-insurance,best_esam_only,0,AUROC,roc-auc,higher,0.9583120395781356,0.9583120395781356,0.9627483366901237,mean,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0,False,1980 +final,homesite-insurance,best_esam_only,1,AUROC,roc-auc,higher,0.9584753793781823,0.9584753793781823,0.9624917932683585,mean,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1,False,1760 +final,homesite-insurance,best_esam_only,2,AUROC,roc-auc,higher,0.9582681142018287,0.9582681142018287,0.9627356393915001,mean,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2,False,1980 +final,homesite-insurance,best_mfb_only,0,AUROC,roc-auc,higher,0.9584246137411132,0.9584246137411132,0.962318960200698,mean,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0,False,1980 +final,homesite-insurance,best_mfb_only,1,AUROC,roc-auc,higher,0.9585282902898349,0.9585282902898349,0.962430602195543,mean,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1,False,1980 +final,homesite-insurance,best_mfb_only,2,AUROC,roc-auc,higher,0.9583243485306635,0.9583243485306635,0.9625461826483815,mean,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2,False,1980 +final,homesite-insurance,best_cf_fisd_only,0,AUROC,roc-auc,higher,0.9589599740471012,0.9589599740471012,0.9628908822909436,mean,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0,False,2420 +final,homesite-insurance,best_cf_fisd_only,1,AUROC,roc-auc,higher,0.9589376508824807,0.9589376508824807,0.9627469836002609,mean,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1,False,1760 +final,homesite-insurance,best_cf_fisd_only,2,AUROC,roc-auc,higher,0.9589488871977169,0.9589488871977169,0.9624131449200803,mean,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2,False,1980 +final,homesite-insurance,best_combined,0,AUROC,roc-auc,higher,0.9589238560635404,0.9589238560635404,0.9624954901031625,mean,exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_combined-evaluation/0,False,1980 +final,homesite-insurance,best_combined,1,AUROC,roc-auc,higher,0.9587836483021374,0.9587836483021374,0.9621208050227469,mean,exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_combined-evaluation/1,False,1760 +final,homesite-insurance,best_combined,2,AUROC,roc-auc,higher,0.9585690329227027,0.9585690329227027,0.9620638665180715,mean,exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_combined-evaluation/2,False,2200 +final,homesite-insurance,baseline_plr,0,AUROC,roc-auc,higher,0.9554763470552776,0.9554763470552776,0.959416932798502,best-head,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml,exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/0,False,1980 +final,homesite-insurance,baseline_plr,1,AUROC,roc-auc,higher,0.9563250405399748,0.9563250405399748,0.9592494275221765,best-head,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml,exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/1,False,1760 +final,homesite-insurance,baseline_plr,2,AUROC,roc-auc,higher,0.9561054927874199,0.9561054927874199,0.9608390423255206,best-head,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml,exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/2,False,1980 +final,homesite-insurance,baseline_plr,0,AUROC,roc-auc,higher,0.9589474980445032,0.9589474980445032,0.9625170670540112,greedy-heads,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0,False,1980 +final,homesite-insurance,baseline_plr,1,AUROC,roc-auc,higher,0.9593694489371631,0.9593694489371631,0.9625145058481993,greedy-heads,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1,False,1760 +final,homesite-insurance,baseline_plr,2,AUROC,roc-auc,higher,0.9591019138539669,0.9591019138539669,0.9629950822915426,greedy-heads,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,False,1980 +final,homesite-insurance,best_rla_only,0,AUROC,roc-auc,higher,0.9560453459701026,0.9560453459701026,0.9600924508313724,best-head,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/0,False,1760 +final,homesite-insurance,best_rla_only,1,AUROC,roc-auc,higher,0.9560549117846388,0.9560549117846388,0.9591263205069757,best-head,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/1,False,1760 +final,homesite-insurance,best_rla_only,2,AUROC,roc-auc,higher,0.9556372338569274,0.9556372338569274,0.9608528752531366,best-head,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/2,False,1980 +final,homesite-insurance,best_rla_only,0,AUROC,roc-auc,higher,0.9591329763748824,0.9591329763748824,0.962370812537228,greedy-heads,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0,False,1760 +final,homesite-insurance,best_rla_only,1,AUROC,roc-auc,higher,0.9590165424760763,0.9590165424760763,0.9620755973239362,greedy-heads,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1,False,1760 +final,homesite-insurance,best_rla_only,2,AUROC,roc-auc,higher,0.9587859694188998,0.9587859694188998,0.9632457301074769,greedy-heads,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2,False,1980 +final,homesite-insurance,best_esam_only,0,AUROC,roc-auc,higher,0.9561512908829329,0.9561512908829329,0.9606910602029306,best-head,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/0,False,1980 +final,homesite-insurance,best_esam_only,1,AUROC,roc-auc,higher,0.9562486459053205,0.9562486459053205,0.9594156280332771,best-head,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/1,False,1760 +final,homesite-insurance,best_esam_only,2,AUROC,roc-auc,higher,0.9550014764588559,0.9550014764588559,0.9593146174587831,best-head,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/2,False,1980 +final,homesite-insurance,best_esam_only,0,AUROC,roc-auc,higher,0.958991133281215,0.958991133281215,0.9627938464179217,greedy-heads,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0,False,1980 +final,homesite-insurance,best_esam_only,1,AUROC,roc-auc,higher,0.9591595812965608,0.9591595812965608,0.9622548696496048,greedy-heads,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1,False,1760 +final,homesite-insurance,best_esam_only,2,AUROC,roc-auc,higher,0.9590920051471822,0.9590920051471822,0.9631468941416912,greedy-heads,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2,False,1980 +final,homesite-insurance,best_mfb_only,0,AUROC,roc-auc,higher,0.9563975490624737,0.9563975490624737,0.9593891219693567,best-head,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/0,False,1980 +final,homesite-insurance,best_mfb_only,1,AUROC,roc-auc,higher,0.9550071913296724,0.9550071913296724,0.9596843251014889,best-head,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/1,False,1980 +final,homesite-insurance,best_mfb_only,2,AUROC,roc-auc,higher,0.9545391258255768,0.9545391258255768,0.9588200872762291,best-head,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/2,False,1980 +final,homesite-insurance,best_mfb_only,0,AUROC,roc-auc,higher,0.9592957095194116,0.9592957095194116,0.9620218603265258,greedy-heads,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0,False,1980 +final,homesite-insurance,best_mfb_only,1,AUROC,roc-auc,higher,0.9591725848256957,0.9591725848256957,0.9622278199335073,greedy-heads,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1,False,1980 +final,homesite-insurance,best_mfb_only,2,AUROC,roc-auc,higher,0.9590785004678369,0.9590785004678369,0.962897309467792,greedy-heads,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2,False,1980 +final,homesite-insurance,best_cf_fisd_only,0,AUROC,roc-auc,higher,0.957813544584969,0.957813544584969,0.961490313471295,best-head,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/0,False,2420 +final,homesite-insurance,best_cf_fisd_only,1,AUROC,roc-auc,higher,0.9580043948939159,0.9580043948939159,0.9620360194454477,best-head,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/1,False,1760 +final,homesite-insurance,best_cf_fisd_only,2,AUROC,roc-auc,higher,0.9575815999587967,0.9575815999587967,0.9611081622343147,best-head,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/2,False,1980 +final,homesite-insurance,best_cf_fisd_only,0,AUROC,roc-auc,higher,0.9594844321379927,0.9594844321379927,0.9634382071404677,greedy-heads,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0,False,2420 +final,homesite-insurance,best_cf_fisd_only,1,AUROC,roc-auc,higher,0.9596519921503349,0.9596519921503349,0.9627588835423583,greedy-heads,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1,False,1760 +final,homesite-insurance,best_cf_fisd_only,2,AUROC,roc-auc,higher,0.9594475404488445,0.9594475404488445,0.9625258379758007,greedy-heads,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2,False,1980 +final,homesite-insurance,best_combined,0,AUROC,roc-auc,higher,0.9576967765860231,0.9576967765860231,0.9617315500639867,best-head,exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/0,False,1980 +final,homesite-insurance,best_combined,1,AUROC,roc-auc,higher,0.9574705028701225,0.9574705028701225,0.9608438143835193,best-head,exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/1,False,1760 +final,homesite-insurance,best_combined,2,AUROC,roc-auc,higher,0.9571173941905297,0.9571173941905297,0.9613875873721536,best-head,exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/2,False,2200 +final,homesite-insurance,best_combined,0,AUROC,roc-auc,higher,0.9597570226838343,0.9597570226838343,0.9633062567165207,greedy-heads,exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml,exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0,False,1980 +final,homesite-insurance,best_combined,1,AUROC,roc-auc,higher,0.9595148440428459,0.9595148440428459,0.9624332721317901,greedy-heads,exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml,exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1,False,1760 +final,homesite-insurance,best_combined,2,AUROC,roc-auc,higher,0.9596270313530297,0.9596270313530297,0.9628075343716237,greedy-heads,exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2,False,2200 +final,cooking-time,baseline_plr,0,RMSE,rmse,lower,0.4627596029926704,-0.4627596029926704,0.4802386655626689,mean,exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml,exp/integrated/final/cooking-time/baseline_plr-evaluation/0,False,23088 +final,cooking-time,baseline_plr,1,RMSE,rmse,lower,0.46274572430329897,-0.46274572430329897,0.4804413950506519,mean,exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml,exp/integrated/final/cooking-time/baseline_plr-evaluation/1,False,19092 +final,cooking-time,baseline_plr,2,RMSE,rmse,lower,0.4626209262038764,-0.4626209262038764,0.48046188039443216,mean,exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml,exp/integrated/final/cooking-time/baseline_plr-evaluation/2,False,28638 +final,cooking-time,best_rla_only,0,RMSE,rmse,lower,0.46272482498106277,-0.46272482498106277,0.4802920784266626,mean,exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_rla_only-evaluation/0,False,23088 +final,cooking-time,best_rla_only,1,RMSE,rmse,lower,0.4627833825636106,-0.4627833825636106,0.4804816670736117,mean,exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_rla_only-evaluation/1,False,19092 +final,cooking-time,best_rla_only,2,RMSE,rmse,lower,0.4626829107835815,-0.4626829107835815,0.4805534877331418,mean,exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_rla_only-evaluation/2,False,28638 +final,cooking-time,best_esam_only,0,RMSE,rmse,lower,0.4627563346148938,-0.4627563346148938,0.4802735094812688,mean,exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_esam_only-evaluation/0,False,23088 +final,cooking-time,best_esam_only,1,RMSE,rmse,lower,0.4627833664641114,-0.4627833664641114,0.4804732935000574,mean,exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_esam_only-evaluation/1,False,19092 +final,cooking-time,best_esam_only,2,RMSE,rmse,lower,0.4626873390860138,-0.4626873390860138,0.4802976784438942,mean,exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_esam_only-evaluation/2,False,29526 +final,cooking-time,best_mfb_only,0,RMSE,rmse,lower,0.4634085546239816,-0.4634085546239816,0.4808915146871742,mean,exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_mfb_only-evaluation/0,False,23088 +final,cooking-time,best_mfb_only,1,RMSE,rmse,lower,0.4632980549483924,-0.4632980549483924,0.48077915988352504,mean,exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_mfb_only-evaluation/1,False,18870 +final,cooking-time,best_mfb_only,2,RMSE,rmse,lower,0.4630663413507467,-0.4630663413507467,0.48078120546833536,mean,exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_mfb_only-evaluation/2,False,28638 +final,cooking-time,best_cf_fisd_only,0,RMSE,rmse,lower,0.4632019090821483,-0.4632019090821483,0.4808496810468957,mean,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0,False,17982 +final,cooking-time,best_cf_fisd_only,1,RMSE,rmse,lower,0.4632797054631699,-0.4632797054631699,0.48088074674610853,mean,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1,False,14652 +final,cooking-time,best_cf_fisd_only,2,RMSE,rmse,lower,0.46327719662519024,-0.46327719662519024,0.48070024325509664,mean,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2,False,12876 +final,cooking-time,best_combined,0,RMSE,rmse,lower,0.4627353230636857,-0.4627353230636857,0.4803077303885999,mean,exp/integrated/final/cooking-time/best_combined-evaluation/0.toml,exp/integrated/final/cooking-time/best_combined-evaluation/0,False,23088 +final,cooking-time,best_combined,1,RMSE,rmse,lower,0.4627920762114039,-0.4627920762114039,0.4804951265119692,mean,exp/integrated/final/cooking-time/best_combined-evaluation/1.toml,exp/integrated/final/cooking-time/best_combined-evaluation/1,False,19092 +final,cooking-time,best_combined,2,RMSE,rmse,lower,0.4627156309124574,-0.4627156309124574,0.4803808798726675,mean,exp/integrated/final/cooking-time/best_combined-evaluation/2.toml,exp/integrated/final/cooking-time/best_combined-evaluation/2,False,29526 +final,cooking-time,baseline_plr,0,RMSE,rmse,lower,0.4652274815891859,-0.4652274815891859,0.4820660657381116,best-head,exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml,exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/0,False,23088 +final,cooking-time,baseline_plr,1,RMSE,rmse,lower,0.46461960653787665,-0.46461960653787665,0.4822480664686614,best-head,exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml,exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/1,False,19092 +final,cooking-time,baseline_plr,2,RMSE,rmse,lower,0.46567157154030486,-0.46567157154030486,0.48281151175783615,best-head,exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml,exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/2,False,28638 +final,cooking-time,baseline_plr,0,RMSE,rmse,lower,0.4625169879783321,-0.4625169879783321,0.47990786370986477,greedy-heads,exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0,False,23088 +final,cooking-time,baseline_plr,1,RMSE,rmse,lower,0.4624852364993238,-0.4624852364993238,0.4800568652741734,greedy-heads,exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1,False,19092 +final,cooking-time,baseline_plr,2,RMSE,rmse,lower,0.46240787039558773,-0.46240787039558773,0.480411417572094,greedy-heads,exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,False,28638 +final,cooking-time,best_rla_only,0,RMSE,rmse,lower,0.4649795846661889,-0.4649795846661889,0.4827695666979872,best-head,exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/0,False,23088 +final,cooking-time,best_rla_only,1,RMSE,rmse,lower,0.46454139300633795,-0.46454139300633795,0.4826743974666991,best-head,exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/1,False,19092 +final,cooking-time,best_rla_only,2,RMSE,rmse,lower,0.46551838172590837,-0.46551838172590837,0.4845241655350321,best-head,exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/2,False,28638 +final,cooking-time,best_rla_only,0,RMSE,rmse,lower,0.4624107384246987,-0.4624107384246987,0.479974942646056,greedy-heads,exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0,False,23088 +final,cooking-time,best_rla_only,1,RMSE,rmse,lower,0.46252858614858405,-0.46252858614858405,0.48029415711103896,greedy-heads,exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1,False,19092 +final,cooking-time,best_rla_only,2,RMSE,rmse,lower,0.462385779530238,-0.462385779530238,0.4801696683856488,greedy-heads,exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2,False,28638 +final,cooking-time,best_esam_only,0,RMSE,rmse,lower,0.46513258351328474,-0.46513258351328474,0.48253251942068065,best-head,exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/0,False,23088 +final,cooking-time,best_esam_only,1,RMSE,rmse,lower,0.46441974054703444,-0.46441974054703444,0.48248829558115747,best-head,exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/1,False,19092 +final,cooking-time,best_esam_only,2,RMSE,rmse,lower,0.4655328979544962,-0.4655328979544962,0.48239242216697287,best-head,exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/2,False,29526 +final,cooking-time,best_esam_only,0,RMSE,rmse,lower,0.46246732196836127,-0.46246732196836127,0.4800169922135662,greedy-heads,exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0,False,23088 +final,cooking-time,best_esam_only,1,RMSE,rmse,lower,0.46256428091435436,-0.46256428091435436,0.4801574722160279,greedy-heads,exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1,False,19092 +final,cooking-time,best_esam_only,2,RMSE,rmse,lower,0.462401054726952,-0.462401054726952,0.48002014306639385,greedy-heads,exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2,False,29526 +final,cooking-time,best_mfb_only,0,RMSE,rmse,lower,0.4652213638497702,-0.4652213638497702,0.48343941871247514,best-head,exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/0,False,23088 +final,cooking-time,best_mfb_only,1,RMSE,rmse,lower,0.4651112788369628,-0.4651112788369628,0.4825797035175412,best-head,exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/1,False,18870 +final,cooking-time,best_mfb_only,2,RMSE,rmse,lower,0.4653975929915622,-0.4653975929915622,0.48330849475451526,best-head,exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/2,False,28638 +final,cooking-time,best_mfb_only,0,RMSE,rmse,lower,0.4624822561623469,-0.4624822561623469,0.4798057758257153,greedy-heads,exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0,False,23088 +final,cooking-time,best_mfb_only,1,RMSE,rmse,lower,0.4626697544496206,-0.4626697544496206,0.47992630708272743,greedy-heads,exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1,False,18870 +final,cooking-time,best_mfb_only,2,RMSE,rmse,lower,0.462357531977379,-0.462357531977379,0.48018644149208806,greedy-heads,exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2,False,28638 +final,cooking-time,best_cf_fisd_only,0,RMSE,rmse,lower,0.4638214410008231,-0.4638214410008231,0.4819659035779245,best-head,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/0,False,17982 +final,cooking-time,best_cf_fisd_only,2,RMSE,rmse,lower,0.46399304740420017,-0.46399304740420017,0.4807545037031361,best-head,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/2,False,12876 +final,cooking-time,best_cf_fisd_only,1,RMSE,rmse,lower,0.4640342331314561,-0.4640342331314561,0.4812057394258983,best-head,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/1,False,14652 +final,cooking-time,best_cf_fisd_only,0,RMSE,rmse,lower,0.4628482268025946,-0.4628482268025946,0.4803119031248436,greedy-heads,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0,False,17982 +final,cooking-time,best_cf_fisd_only,2,RMSE,rmse,lower,0.46301623543457804,-0.46301623543457804,0.4803092971079454,greedy-heads,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2,False,12876 +final,cooking-time,best_cf_fisd_only,1,RMSE,rmse,lower,0.4629984701854322,-0.4629984701854322,0.4803911006799199,greedy-heads,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1,False,14652 +final,cooking-time,best_combined,0,RMSE,rmse,lower,0.46492255368357066,-0.46492255368357066,0.4825495500767423,best-head,exp/integrated/final/cooking-time/best_combined-evaluation/0.toml,exp/integrated/final/cooking-time/best_combined-best-head-evaluation/0,False,23088 +final,cooking-time,best_combined,1,RMSE,rmse,lower,0.46452431161307095,-0.46452431161307095,0.48248902135425475,best-head,exp/integrated/final/cooking-time/best_combined-evaluation/1.toml,exp/integrated/final/cooking-time/best_combined-best-head-evaluation/1,False,19092 +final,cooking-time,best_combined,2,RMSE,rmse,lower,0.46544124766603506,-0.46544124766603506,0.4832114117959526,best-head,exp/integrated/final/cooking-time/best_combined-evaluation/2.toml,exp/integrated/final/cooking-time/best_combined-best-head-evaluation/2,False,29526 +final,cooking-time,best_combined,0,RMSE,rmse,lower,0.4624234187671886,-0.4624234187671886,0.4800586966547807,greedy-heads,exp/integrated/final/cooking-time/best_combined-evaluation/0.toml,exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0,False,23088 +final,cooking-time,best_combined,1,RMSE,rmse,lower,0.46250972286520653,-0.46250972286520653,0.4801020579816987,greedy-heads,exp/integrated/final/cooking-time/best_combined-evaluation/1.toml,exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1,False,19092 +final,cooking-time,best_combined,2,RMSE,rmse,lower,0.4623945128795896,-0.4623945128795896,0.4799174115040007,greedy-heads,exp/integrated/final/cooking-time/best_combined-evaluation/2.toml,exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2,False,29526 +final,delivery-eta,baseline_plr,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0,False,2184 +final,delivery-eta,baseline_plr,1,RMSE,rmse,lower,0.5562247345576334,-0.5562247345576334,0.550284872190118,mean,exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml,exp/integrated/final/delivery-eta/baseline_plr-evaluation/1,False,1638 +final,delivery-eta,baseline_plr,2,RMSE,rmse,lower,0.5557493871736087,-0.5557493871736087,0.5507763801521746,mean,exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml,exp/integrated/final/delivery-eta/baseline_plr-evaluation/2,False,1638 +final,delivery-eta,best_rla_only,0,RMSE,rmse,lower,0.5556336245387727,-0.5556336245387727,0.5518591323593222,mean,exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_rla_only-evaluation/0,False,1365 +final,delivery-eta,best_rla_only,1,RMSE,rmse,lower,0.5562834011722252,-0.5562834011722252,0.5505008118837418,mean,exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_rla_only-evaluation/1,False,1365 +final,delivery-eta,best_rla_only,2,RMSE,rmse,lower,0.5564119371157284,-0.5564119371157284,0.5491394300416609,mean,exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_rla_only-evaluation/2,False,1365 +final,delivery-eta,best_esam_only,0,RMSE,rmse,lower,0.5556360918188752,-0.5556360918188752,0.5494649592968895,mean,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0,False,2184 +final,delivery-eta,best_esam_only,1,RMSE,rmse,lower,0.5559996009180297,-0.5559996009180297,0.5517844674752832,mean,exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_esam_only-evaluation/1,False,1638 +final,delivery-eta,best_esam_only,2,RMSE,rmse,lower,0.555909194911898,-0.555909194911898,0.5492662194956084,mean,exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_esam_only-evaluation/2,False,1638 +final,delivery-eta,best_mfb_only,0,RMSE,rmse,lower,0.5576335149669441,-0.5576335149669441,0.5507854163911843,mean,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0,False,2730 +final,delivery-eta,best_mfb_only,1,RMSE,rmse,lower,0.5557639999222178,-0.5557639999222178,0.5537457867240355,mean,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1,False,1638 +final,delivery-eta,best_mfb_only,2,RMSE,rmse,lower,0.556207481645197,-0.556207481645197,0.551272504610975,mean,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2,False,3003 +final,delivery-eta,best_cf_fisd_only,0,RMSE,rmse,lower,0.5562445854609275,-0.5562445854609275,0.5504166498793033,mean,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0,False,2184 +final,delivery-eta,best_cf_fisd_only,1,RMSE,rmse,lower,0.5564364677950075,-0.5564364677950075,0.550973168882935,mean,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1,False,1365 +final,delivery-eta,best_cf_fisd_only,2,RMSE,rmse,lower,0.5569470014167281,-0.5569470014167281,0.5474489469980929,mean,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2,False,1365 +final,delivery-eta,best_combined,0,RMSE,rmse,lower,0.5559417888696784,-0.5559417888696784,0.5541331785404355,mean,exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml,exp/integrated/final/delivery-eta/best_combined-evaluation/0,False,1365 +final,delivery-eta,best_combined,1,RMSE,rmse,lower,0.5563776030607058,-0.5563776030607058,0.5504132928776152,mean,exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml,exp/integrated/final/delivery-eta/best_combined-evaluation/1,False,1638 +final,delivery-eta,best_combined,2,RMSE,rmse,lower,0.556831808887752,-0.556831808887752,0.5516551774702309,mean,exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml,exp/integrated/final/delivery-eta/best_combined-evaluation/2,False,819 +final,delivery-eta,baseline_plr,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml,exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/0,False,2184 +final,delivery-eta,baseline_plr,1,RMSE,rmse,lower,0.5578575885405295,-0.5578575885405295,0.5568376961917461,best-head,exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml,exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/1,False,1638 +final,delivery-eta,baseline_plr,2,RMSE,rmse,lower,0.5576450053690213,-0.5576450053690213,0.5574656479348947,best-head,exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml,exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/2,False,1638 +final,delivery-eta,baseline_plr,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml,exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0,False,2184 +final,delivery-eta,baseline_plr,1,RMSE,rmse,lower,0.5553836483690917,-0.5553836483690917,0.5515298017800222,greedy-heads,exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml,exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1,False,1638 +final,delivery-eta,baseline_plr,2,RMSE,rmse,lower,0.5554437183690072,-0.5554437183690072,0.5516331354574737,greedy-heads,exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml,exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2,False,1638 +final,delivery-eta,best_rla_only,0,RMSE,rmse,lower,0.5574634025946379,-0.5574634025946379,0.5547066537478277,best-head,exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0,False,1365 +final,delivery-eta,best_rla_only,1,RMSE,rmse,lower,0.5576988200546102,-0.5576988200546102,0.551726996987587,best-head,exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1,False,1365 +final,delivery-eta,best_rla_only,2,RMSE,rmse,lower,0.5580165521873937,-0.5580165521873937,0.5499792279309027,best-head,exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2,False,1365 +final,delivery-eta,best_rla_only,0,RMSE,rmse,lower,0.5552720495549479,-0.5552720495549479,0.5524877260752137,greedy-heads,exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0,False,1365 +final,delivery-eta,best_rla_only,1,RMSE,rmse,lower,0.55584140086384,-0.55584140086384,0.5514378793216401,greedy-heads,exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1,False,1365 +final,delivery-eta,best_rla_only,2,RMSE,rmse,lower,0.555843679564389,-0.555843679564389,0.5502940789713319,greedy-heads,exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2,False,1365 +final,delivery-eta,best_esam_only,0,RMSE,rmse,lower,0.5582863288518123,-0.5582863288518123,0.5560390501031047,best-head,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/0,False,2184 +final,delivery-eta,best_esam_only,1,RMSE,rmse,lower,0.5579506166150647,-0.5579506166150647,0.5600144253303337,best-head,exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/1,False,1638 +final,delivery-eta,best_esam_only,2,RMSE,rmse,lower,0.5578059530007539,-0.5578059530007539,0.5592548752846349,best-head,exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/2,False,1638 +final,delivery-eta,best_esam_only,0,RMSE,rmse,lower,0.5551513290724005,-0.5551513290724005,0.5507488105906536,greedy-heads,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0,False,2184 +final,delivery-eta,best_esam_only,1,RMSE,rmse,lower,0.5557057343261759,-0.5557057343261759,0.5517152752978882,greedy-heads,exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1,False,1638 +final,delivery-eta,best_esam_only,2,RMSE,rmse,lower,0.5554111756694609,-0.5554111756694609,0.5509796867295551,greedy-heads,exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2,False,1638 +final,delivery-eta,best_mfb_only,0,RMSE,rmse,lower,0.5595885535124743,-0.5595885535124743,0.5534224254724212,best-head,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/0,False,2730 +final,delivery-eta,best_mfb_only,1,RMSE,rmse,lower,0.5585395946793884,-0.5585395946793884,0.5622948696369855,best-head,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/1,False,1638 +final,delivery-eta,best_mfb_only,2,RMSE,rmse,lower,0.5610582099606907,-0.5610582099606907,0.5524986222625401,best-head,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/2,False,3003 +final,delivery-eta,best_mfb_only,0,RMSE,rmse,lower,0.5551008108444078,-0.5551008108444078,0.5501919833980724,greedy-heads,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0,False,2730 +final,delivery-eta,best_mfb_only,1,RMSE,rmse,lower,0.5549583042651809,-0.5549583042651809,0.5533328640985606,greedy-heads,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1,False,1638 +final,delivery-eta,best_mfb_only,2,RMSE,rmse,lower,0.55517701590878,-0.55517701590878,0.5505736480564649,greedy-heads,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2,False,3003 +final,delivery-eta,best_cf_fisd_only,0,RMSE,rmse,lower,0.557661572508718,-0.557661572508718,0.553144673358601,best-head,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/0,False,2184 +final,delivery-eta,best_cf_fisd_only,1,RMSE,rmse,lower,0.5570080264093609,-0.5570080264093609,0.5540194450901057,best-head,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/1,False,1365 +final,delivery-eta,best_cf_fisd_only,2,RMSE,rmse,lower,0.5578165315968546,-0.5578165315968546,0.5527351028922852,best-head,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/2,False,1365 +final,delivery-eta,best_cf_fisd_only,0,RMSE,rmse,lower,0.5556691040453957,-0.5556691040453957,0.5521822756750591,greedy-heads,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0,False,2184 +final,delivery-eta,best_cf_fisd_only,1,RMSE,rmse,lower,0.5559175580175352,-0.5559175580175352,0.5539339615490284,greedy-heads,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1,False,1365 +final,delivery-eta,best_cf_fisd_only,2,RMSE,rmse,lower,0.5559178796729317,-0.5559178796729317,0.551186027318127,greedy-heads,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2,False,1365 +final,delivery-eta,best_combined,0,RMSE,rmse,lower,0.5572907783524989,-0.5572907783524989,0.5545303498210034,best-head,exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml,exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0,False,1365 +final,delivery-eta,best_combined,2,RMSE,rmse,lower,0.5577944659141506,-0.5577944659141506,0.5536175199956386,best-head,exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml,exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2,False,819 +final,delivery-eta,best_combined,1,RMSE,rmse,lower,0.5571610811723605,-0.5571610811723605,0.5529389828325415,best-head,exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml,exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1,False,1638 +final,delivery-eta,best_combined,0,RMSE,rmse,lower,0.5552988041877751,-0.5552988041877751,0.5525187688548042,greedy-heads,exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml,exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0,False,1365 +final,delivery-eta,best_combined,2,RMSE,rmse,lower,0.5564955940706238,-0.5564955940706238,0.5523659114897095,greedy-heads,exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml,exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2,False,819 +final,delivery-eta,best_combined,1,RMSE,rmse,lower,0.5558474595058753,-0.5558474595058753,0.5521241448870569,greedy-heads,exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml,exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1,False,1638 diff --git a/tabm_integrated/paper/exp/final_integrated_summary.csv b/tabm_integrated/paper/exp/final_integrated_summary.csv index 582389d43..2de242084 100644 --- a/tabm_integrated/paper/exp/final_integrated_summary.csv +++ b/tabm_integrated/paper/exp/final_integrated_summary.csv @@ -1,51 +1,31 @@ -dataset,variant,metric,metric_direction,inference_mode,precision,n_seeds,mean,std,baseline_mean,absolute_delta,percent_delta,safe_claim_status,config_path,result_path -sberbank-housing,baseline_plr,rmse,lower,mean,bfloat16,3,11.50568931672913,0.043202216269019954,11.50568931672913,0.0,0.0,baseline,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation -sberbank-housing,rla_only,rmse,lower,mean,bfloat16,3,11.50411760953292,0.00470548519967073,11.50568931672913,0.0015717071962111362,0.013660261049513072,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation -sberbank-housing,esam_only,rmse,lower,mean,bfloat16,3,11.506508100749382,0.04324706873702009,11.50568931672913,-0.0008187840202520391,-0.007116340427005423,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation -sberbank-housing,mfb_only,rmse,lower,mean,bfloat16,3,11.525666016611616,0.011106780774725373,11.50568931672913,-0.01997669988248596,-0.17362453767494038,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation -sberbank-housing,cf_fisd_only,rmse,lower,mean,bfloat16,3,11.535485883166638,0.005402700092330267,11.50568931672913,-0.02979656643750772,-0.258972458035903,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation -sberbank-housing,all_four_combined,rmse,lower,mean,bfloat16,3,11.535750290176733,0.005151538366452624,11.50568931672913,-0.03006097344760228,-0.26127051252717204,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation -sberbank-housing,all_minus_rla,rmse,lower,mean,bfloat16,3,11.535163696421723,0.004119612327761725,11.50568931672913,-0.02947437969259248,-0.25617221951002184,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation -sberbank-housing,all_minus_esam,rmse,lower,mean,bfloat16,3,11.534415356648143,0.004316423799383681,11.50568931672913,-0.02872603991901279,-0.24966813485260272,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation -sberbank-housing,all_minus_mfb,rmse,lower,mean,bfloat16,3,11.531460473260475,0.004494970895064652,11.50568931672913,-0.025771156531344985,-0.2239862021467418,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation -sberbank-housing,all_minus_cf_fisd,rmse,lower,mean,bfloat16,3,11.511691772117072,0.015408095541226104,11.50568931672913,-0.006002455387941197,-0.0521694548036657,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation -ecom-offers,baseline_plr,roc-auc,higher,mean,bfloat16,3,0.576461278688309,0.0003009557265118819,0.576461278688309,0.0,0.0,baseline,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation -ecom-offers,rla_only,roc-auc,higher,mean,bfloat16,3,0.5768068923109811,0.00019864992134586008,0.576461278688309,0.00034561362267215845,0.05995435174042813,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation -ecom-offers,esam_only,roc-auc,higher,mean,bfloat16,3,0.5765998904958146,0.0001698137408662012,0.576461278688309,0.00013861180750562863,0.024045293696226846,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation -ecom-offers,mfb_only,roc-auc,higher,mean,bfloat16,3,0.5777060181541288,0.003396500904841115,0.576461278688309,0.0012447394658198219,0.215927680112726,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation -ecom-offers,cf_fisd_only,roc-auc,higher,mean,bfloat16,3,0.5733136336223631,0.00039242840319386274,0.576461278688309,-0.003147645065945892,-0.5460288803973277,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation -ecom-offers,all_four_combined,roc-auc,higher,mean,bfloat16,3,0.5753567080016927,0.0017816599244741948,0.576461278688309,-0.0011045706866162908,-0.19161229512754993,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation -ecom-offers,all_minus_rla,roc-auc,higher,mean,bfloat16,3,0.5720367783866491,0.002025737566045578,0.576461278688309,-0.004424500301659862,-0.7675277534212626,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation -ecom-offers,all_minus_esam,roc-auc,higher,mean,bfloat16,3,0.5720048144912337,0.0010919402972391877,0.576461278688309,-0.004456464197075238,-0.7730726003341564,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation -ecom-offers,all_minus_mfb,roc-auc,higher,mean,bfloat16,3,0.5743689329219358,0.00047029544330458393,0.576461278688309,-0.0020923457663731826,-0.3629638006448839,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation -ecom-offers,all_minus_cf_fisd,roc-auc,higher,mean,bfloat16,3,0.5767558170270664,0.0034835078449268714,0.576461278688309,0.00029453833875736635,0.051094210425991576,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation -homesite-insurance,baseline_plr,roc-auc,higher,mean,bfloat16,3,0.953230710648917,0.0006004532712494962,0.953230710648917,0.0,0.0,baseline,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation -homesite-insurance,rla_only,roc-auc,higher,mean,bfloat16,3,0.9519622332000766,0.0007320522349105445,0.953230710648917,-0.0012684774488403994,-0.13307139968002882,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation -homesite-insurance,esam_only,roc-auc,higher,mean,bfloat16,3,0.9532752861003813,0.0005403018859361379,0.953230710648917,4.457545146430153e-05,0.004676250037512593,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation -homesite-insurance,mfb_only,roc-auc,higher,mean,bfloat16,3,0.9500351513416514,0.00033536387160697765,0.953230710648917,-0.003195559307265561,-0.3352346154573814,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation -homesite-insurance,cf_fisd_only,roc-auc,higher,mean,bfloat16,3,0.9543647086844239,0.0004525369838386773,0.953230710648917,0.0011339980355069557,0.11896364886680795,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation -homesite-insurance,all_four_combined,roc-auc,higher,mean,bfloat16,3,0.9495877457352138,0.0007166592249211433,0.953230710648917,-0.003642964913703195,-0.3821703259248988,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation -homesite-insurance,all_minus_rla,roc-auc,higher,mean,bfloat16,3,0.9504702744359411,0.0004745605385127754,0.953230710648917,-0.0027604362129758364,-0.28958741909360586,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation -homesite-insurance,all_minus_esam,roc-auc,higher,mean,bfloat16,3,0.9494433074194073,0.0015262231096144455,0.953230710648917,-0.003787403229509656,-0.3973228293212837,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation -homesite-insurance,all_minus_mfb,roc-auc,higher,mean,bfloat16,3,0.9533196440909745,0.0005128632590870418,0.953230710648917,8.893344205751497e-05,0.009329687038405744,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation -homesite-insurance,all_minus_cf_fisd,roc-auc,higher,mean,bfloat16,3,0.9459095199440131,0.0017637792250980726,0.953230710648917,-0.0073211907049038905,-0.7680397434866477,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation -cooking-time,baseline_plr,rmse,lower,mean,bfloat16,3,2.564567532647581,0.0028960770983219863,2.564567532647581,0.0,0.0,baseline,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation -cooking-time,rla_only,rmse,lower,mean,bfloat16,3,2.5608798043212375,0.013833168648965952,2.564567532647581,0.0036877283263434535,0.1437953292084438,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation -cooking-time,esam_only,rmse,lower,mean,bfloat16,3,2.56386112751868,0.0027853572581009772,2.564567532647581,0.0007064051289011886,0.02754480511464315,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation -cooking-time,mfb_only,rmse,lower,mean,bfloat16,3,2.544497473583304,0.009546779475477335,2.564567532647581,0.020070059064277057,0.7825903903399004,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation -cooking-time,cf_fisd_only,rmse,lower,mean,bfloat16,3,2.5632362689318433,0.002785603213876169,2.564567532647581,0.0013312637157376628,0.05190987169533831,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation -cooking-time,all_four_combined,rmse,lower,mean,bfloat16,3,2.5442129846116677,0.006841179137786861,2.564567532647581,0.0203545480359133,0.793683448643674,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation -cooking-time,all_minus_rla,rmse,lower,mean,bfloat16,3,2.54692834589173,0.0032536080083122717,2.564567532647581,0.01763918675585119,0.68780355874041,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation -cooking-time,all_minus_esam,rmse,lower,mean,bfloat16,3,2.543215475377082,0.0047318016724798495,2.564567532647581,0.021352057270498825,0.8325792555150855,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation -cooking-time,all_minus_mfb,rmse,lower,mean,bfloat16,3,2.559927458455293,0.012512315486430373,2.564567532647581,0.004640074192288068,0.18093008404804212,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation -cooking-time,all_minus_cf_fisd,rmse,lower,mean,bfloat16,3,2.545176983167345,0.002904437047405368,2.564567532647581,0.019390549480235997,0.7560943212993806,win,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation -delivery-eta,baseline_plr,rmse,lower,mean,bfloat16,3,1.8542282946096622,0.012900047232982671,1.8542282946096622,0.0,0.0,baseline,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation -delivery-eta,rla_only,rmse,lower,mean,bfloat16,3,1.879132189229867,0.004746592370465458,1.8542282946096622,-0.024903894620204703,-1.3430867543441989,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation -delivery-eta,esam_only,rmse,lower,mean,bfloat16,3,1.8564487022757508,0.013392767875906186,1.8542282946096622,-0.002220407666088553,-0.11974834342369778,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation -delivery-eta,mfb_only,rmse,lower,mean,bfloat16,3,1.8881700161777082,0.01022580688177479,1.8542282946096622,-0.03394172156804598,-1.8305039172747133,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation -delivery-eta,cf_fisd_only,rmse,lower,mean,bfloat16,3,1.8593434631671701,0.013321584910823822,1.8542282946096622,-0.005115168557507932,-0.27586509020372474,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation -delivery-eta,all_four_combined,rmse,lower,mean,bfloat16,3,1.8809046331968828,0.015428261901006317,1.8542282946096622,-0.026676338587220627,-1.438676060804925,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation -delivery-eta,all_minus_rla,rmse,lower,mean,bfloat16,3,1.8790456474067045,0.024307524600509958,1.8542282946096622,-0.024817352797042336,-1.3384194853021965,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation -delivery-eta,all_minus_esam,rmse,lower,mean,bfloat16,3,1.878911501253891,0.018605047649282478,1.8542282946096622,-0.02468320664422885,-1.3311848770717292,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation -delivery-eta,all_minus_mfb,rmse,lower,mean,bfloat16,3,1.8687456723489537,0.0067834487351692615,1.8542282946096622,-0.014517377739291515,-0.7829336755077185,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation -delivery-eta,all_minus_cf_fisd,rmse,lower,mean,bfloat16,3,1.872915155877866,0.01624074888470241,1.8542282946096622,-0.018686861268203847,-1.0077972233800725,loss,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0.toml,/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation +dataset,variant,source_variant,metric,direction,validation_metric,test_metric,mean,std,n_seeds,baseline_mean,absolute_delta,percent_delta,precision,inference_mode,config_path,result_path,status +sberbank-housing,baseline_plr,baseline_plr,RMSE,lower,0.23552481981288087,0.23441786078206484,0.23441786078206484,0.0035482159973780638,3,0.23441786078206484,0.0,0.0,3 seeds,mean,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2,baseline +sberbank-housing,best_rla_only,rla_rank2_noise0.001,RMSE,lower,0.23385412997377952,0.2323679570932143,0.2323679570932143,0.002285510337847783,3,0.23441786078206484,0.002049903688850524,0.8744656580397225,3 seeds,greedy-heads,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2,weak_win +sberbank-housing,best_esam_only,esam_rho0.0025,RMSE,lower,0.23381482399811443,0.23440607616465495,0.23440607616465495,0.004011724082624796,3,0.23441786078206484,1.1784617409887899e-05,0.005027184093640331,3 seeds,greedy-heads,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2,weak_win +sberbank-housing,best_mfb_only,mfb_keep0.8,RMSE,lower,0.23321906081613947,0.23509963234240183,0.23509963234240183,0.003956610803973734,3,0.23441786078206484,-0.0006817715603369967,-0.29083601311882573,3 seeds,greedy-heads,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2,loss +sberbank-housing,best_cf_fisd_only,cf_fisd_only,RMSE,lower,0.23407921163352594,0.23309306252476805,0.23309306252476805,0.0005850816922811394,3,0.23441786078206484,0.0013247982572967831,0.5651439070713261,3 seeds,greedy-heads,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2,weak_win +sberbank-housing,best_combined,mfb_cf_fisd,RMSE,lower,0.23329192884803412,0.23497877374993395,0.23497877374993395,0.003370630932382365,3,0.23441786078206484,-0.0005609129678691149,-0.2392791086813083,3 seeds,greedy-heads,exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2,loss +ecom-offers,baseline_plr,baseline_plr,AUROC,higher,0.6214296509780997,0.5900580480375381,0.5900580480375381,0.0009703638630149601,3,0.5900580480375381,0.0,0.0,3 seeds,mean,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2,baseline +ecom-offers,best_rla_only,rla_rank2_noise0.0001,AUROC,higher,0.6348279404183594,0.5984469268269463,0.5984469268269463,0.002797349140724218,3,0.5900580480375381,0.008388878789408194,1.4217039861262113,3 seeds,best-head,exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2,clear_win +ecom-offers,best_esam_only,esam_rho0.001,AUROC,higher,0.6366563506545114,0.6004196058355985,0.6004196058355985,0.003493353451214683,3,0.5900580480375381,0.010361557798060361,1.7560234679488995,3 seeds,greedy-heads,exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2,clear_win +ecom-offers,best_mfb_only,mfb_keep0.7,AUROC,higher,0.6539769427777475,0.6095164790194458,0.6095164790194458,0.0020894834660806928,3,0.5900580480375381,0.019458430981907693,3.2977146988544748,3 seeds,best-head,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2,clear_win +ecom-offers,best_cf_fisd_only,cf_fisd_lambda0.05,AUROC,higher,0.636440295793426,0.6002048966489212,0.6002048966489212,0.0034582462660955543,3,0.5900580480375381,0.010146848611383041,1.719635660445652,3 seeds,greedy-heads,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2,clear_win +ecom-offers,best_combined,mfb_cf_fisd,AUROC,higher,0.6485201184226032,0.6052109584242933,0.6052109584242933,0.003222479623952224,3,0.5900580480375381,0.015152910386755147,2.5680372358536414,3 seeds,greedy-heads,exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml,exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2,clear_win +homesite-insurance,baseline_plr,baseline_plr,AUROC,higher,0.9584198249724393,0.9624941853379376,0.9624941853379376,0.0001780794332642721,3,0.9624941853379376,0.0,0.0,3 seeds,mean,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2,baseline +homesite-insurance,best_rla_only,rla_rank4_noise1e-05,AUROC,higher,0.9589784960899528,0.9625640466562138,0.9625640466562138,0.0006085287596608143,3,0.9624941853379376,6.986131827613207e-05,0.007258362631209385,3 seeds,greedy-heads,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2,weak_win +homesite-insurance,best_esam_only,esam_only,AUROC,higher,0.959080906574986,0.9627318700697393,0.9627318700697393,0.0004492301463849077,3,0.9624941853379376,0.00023768473180163507,0.024694666775382387,3 seeds,greedy-heads,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2,clear_win +homesite-insurance,best_mfb_only,mfb_keep0.95,AUROC,higher,0.959182264937648,0.962382329909275,0.962382329909275,0.0004577202193422504,3,0.9624941853379376,-0.00011185542866265497,-0.011621413445046615,3 seeds,greedy-heads,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2,loss +homesite-insurance,best_cf_fisd_only,cf_fisd_only,AUROC,higher,0.959527988245724,0.9629076428862089,0.9629076428862089,0.00047402677133884024,3,0.9624941853379376,0.0004134575482712677,0.04295688790328642,3 seeds,greedy-heads,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2,clear_win +homesite-insurance,best_combined,mfb_cf_fisd,AUROC,higher,0.95963296602657,0.9628490210733115,0.9628490210733115,0.0004379684704485265,3,0.9624941853379376,0.0003548357353738707,0.03686627314525393,3 seeds,greedy-heads,exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2,clear_win +cooking-time,baseline_plr,baseline_plr,RMSE,lower,0.46270875116661525,0.4803806470025843,0.4803806470025843,0.00012338540954176603,3,0.4803806470025843,0.0,0.0,3 seeds,mean,exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml,exp/integrated/final/cooking-time/baseline_plr-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-evaluation/2,baseline +cooking-time,best_rla_only,rla_rank4_noise0.001,RMSE,lower,0.46244170136784024,0.48014625604758127,0.48014625604758127,0.00016088993704992786,3,0.4803806470025843,0.00023439095500304985,0.04879275559196058,3 seeds,greedy-heads,exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2,clear_win +cooking-time,best_esam_only,esam_rho0.0025,RMSE,lower,0.46247755253655587,0.4800648691653293,0.4800648691653293,8.021206715574865e-05,3,0.4803806470025843,0.00031577783725500463,0.0657349206770409,3 seeds,greedy-heads,exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2,clear_win +cooking-time,best_mfb_only,mfb_keep0.8,RMSE,lower,0.4625031808631155,0.4799728414668436,0.4799728414668436,0.00019455249694504936,3,0.4803806470025843,0.0004078055357407129,0.0848921658866326,3 seeds,greedy-heads,exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2,clear_win +cooking-time,best_cf_fisd_only,cf_fisd_lambda0.05,RMSE,lower,0.46295431080753496,0.48033743363756964,0.48033743363756964,4.649528370692734e-05,3,0.4803806470025843,4.3213365014671545e-05,0.008995650695819781,3 seeds,greedy-heads,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2,weak_win +cooking-time,best_combined,rla_esam,RMSE,lower,0.46244255150399494,0.48002605538016,0.48002605538016,9.65539747406379e-05,3,0.4803806470025843,0.000354591622424294,0.07381471852307706,3 seeds,greedy-heads,exp/integrated/final/cooking-time/best_combined-evaluation/0.toml;exp/integrated/final/cooking-time/best_combined-evaluation/1.toml;exp/integrated/final/cooking-time/best_combined-evaluation/2.toml,exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2,clear_win +delivery-eta,baseline_plr,baseline_plr,RMSE,lower,0.5559497041253089,0.5502258480608934,0.5502258480608934,0.0005822921162605647,3,0.5502258480608934,0.0,0.0,3 seeds,mean,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2,baseline +delivery-eta,best_rla_only,rla_rank2_noise0.001,RMSE,lower,0.5556523766610589,0.5514065614560619,0.5514065614560619,0.001097158835638919,3,0.5502258480608934,-0.0011807133951685644,-0.21458704627011546,3 seeds,greedy-heads,exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2,loss +delivery-eta,best_esam_only,esam_only,RMSE,lower,0.5554227463560124,0.5511479242060323,0.5511479242060323,0.0005047191238214393,3,0.5502258480608934,-0.0009220761451389503,-0.16758139378375847,3 seeds,greedy-heads,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2,loss +delivery-eta,best_mfb_only,mfb_keep0.7,RMSE,lower,0.5550787103394562,0.551366165184366,0.551366165184366,0.0017138685603590752,3,0.5502258480608934,-0.0011403171234726228,-0.20724528436665232,3 seeds,greedy-heads,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2,loss +delivery-eta,best_cf_fisd_only,cf_fisd_lambda0.05,RMSE,lower,0.5558348472452875,0.5524340881807381,0.5524340881807381,0.0013911659814170834,3,0.5502258480608934,-0.002208240119844773,-0.40133340293391445,3 seeds,greedy-heads,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2,loss +delivery-eta,best_combined,rla_esam,RMSE,lower,0.5558806192547581,0.5523362750771902,0.5523362750771902,0.00019897426137676394,3,0.5502258480608934,-0.002110427016296823,-0.3835565020680858,3 seeds,greedy-heads,exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml,exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2,loss diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c9ff6ead7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,147 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44650602179756516, + "mae": 0.34757497906684875, + "r2": 0.5420529246330261, + "score": -0.44650602179756516 + }, + "val": { + "rmse": 0.4652274815891859, + "mae": 0.3617282807826996, + "r2": 0.503640353679657, + "score": -0.4652274815891859 + }, + "test": { + "rmse": 0.4820660657381116, + "mae": 0.3761262893676758, + "r2": 0.48187607526779175, + "score": -0.4820660657381116 + } + }, + "time": "0:09:50.419637", + "diversity": { + "train": { + "mean_centered_corr": -0.020426616072654724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051189813762903214 + }, + "val": { + "mean_centered_corr": -0.02039431221783161, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05273108929395676 + }, + "test": { + "mean_centered_corr": -0.020482471212744713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053407639265060425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27 + ], + "head_selection_time": "0:00:00.027737" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..78c613dfb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..21d650115 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0/report.json @@ -0,0 +1,143 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44389690465868414, + "mae": 0.34541288018226624, + "r2": 0.5473892688751221, + "score": -0.44389690465868414 + }, + "val": { + "rmse": 0.4627596029926704, + "mae": 0.35959550738334656, + "r2": 0.5088924169540405, + "score": -0.4627596029926704 + }, + "test": { + "rmse": 0.4802386655626689, + "mae": 0.37469902634620667, + "r2": 0.4857967495918274, + "score": -0.4802386655626689 + } + }, + "time": "0:09:50.419637", + "diversity": { + "train": { + "mean_centered_corr": -0.020426616072654724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051189813762903214 + }, + "val": { + "mean_centered_corr": -0.02039431221783161, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05273108929395676 + }, + "test": { + "mean_centered_corr": -0.020482471212744713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053407639265060425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9addb5172 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/cooking-time/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44394242184625543, + "mae": 0.3455282151699066, + "r2": 0.547296404838562, + "score": -0.44394242184625543 + }, + "val": { + "rmse": 0.4625169879783321, + "mae": 0.3595449924468994, + "r2": 0.5094072818756104, + "score": -0.4625169879783321 + }, + "test": { + "rmse": 0.47990786370986477, + "mae": 0.3745970129966736, + "r2": 0.4865049123764038, + "score": -0.47990786370986477 + } + }, + "time": "0:09:50.419637", + "diversity": { + "train": { + "mean_centered_corr": -0.020426616072654724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051189813762903214 + }, + "val": { + "mean_centered_corr": -0.02039431221783161, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05273108929395676 + }, + "test": { + "mean_centered_corr": -0.020482471212744713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053407639265060425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27, + 17, + 5, + 3, + 14, + 4, + 22, + 6, + 24, + 10, + 16 + ], + "head_selection_time": "0:00:00.401807" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4fd0f8dbc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,168 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:15.524601", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023608" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..d07370fc8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..95e980416 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0/report.json @@ -0,0 +1,164 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:15.524601", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b9623297c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,181 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:15.524601", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.441754" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..588061229 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7835229427379597, + "recall": 0.9817888071846897, + "f1-score": 0.871522047464133, + "support": 84179.0 + }, + "1": { + "precision": 0.602952602952603, + "recall": 0.09252046737143312, + "f1-score": 0.1604244909209937, + "support": 25162.0 + }, + "accuracy": 0.7771467244674916, + "macro avg": { + "precision": 0.6932377728452814, + "recall": 0.5371546372780615, + "f1-score": 0.5159732691925634, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7419693545168977, + "recall": 0.7771467244674916, + "f1-score": 0.707881357167369, + "support": 109341.0 + }, + "cross-entropy": 0.49822654924283316, + "roc-auc": 0.6867900811541505, + "score": 0.6867900811541505 + }, + "val": { + "0": { + "precision": 0.6877843221759325, + "recall": 0.9255703054517335, + "f1-score": 0.7891541441169199, + "support": 15518.0 + }, + "1": { + "precision": 0.6580817051509769, + "recall": 0.2542605512981814, + "f1-score": 0.36680141902483293, + "support": 8743.0 + }, + "accuracy": 0.6836486542187049, + "macro avg": { + "precision": 0.6729330136634547, + "recall": 0.5899154283749575, + "f1-score": 0.5779777815708764, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6770803124216278, + "recall": 0.6836486542187049, + "f1-score": 0.6369497883409784, + "support": 24261.0 + }, + "cross-entropy": 0.6337455659020594, + "roc-auc": 0.6404461738890127, + "score": 0.6404461738890127 + }, + "test": { + "0": { + "precision": 0.6663787430075683, + "recall": 0.9573927431745657, + "f1-score": 0.785807828491051, + "support": 16922.0 + }, + "1": { + "precision": 0.6635557629491368, + "recall": 0.14916605475715933, + "f1-score": 0.24357656731757452, + "support": 9533.0 + }, + "accuracy": 0.6661500661500661, + "macro avg": { + "precision": 0.6649672529783526, + "recall": 0.5532793989658625, + "f1-score": 0.5146921979043128, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6653614884660062, + "recall": 0.6661500661500661, + "f1-score": 0.5904160079366473, + "support": 26455.0 + }, + "cross-entropy": 0.6685406788276543, + "roc-auc": 0.6034781946000056, + "score": 0.6034781946000056 + } + }, + "time": "0:01:00.962097", + "diversity": { + "train": { + "mean_centered_corr": -0.013087093830108643, + "mean_pairwise_disagreement": 0.025822423398123243, + "member_std": 0.02709772437810898 + }, + "val": { + "mean_centered_corr": -0.004288136959075928, + "mean_pairwise_disagreement": 0.08692423855623854, + "member_std": 0.060995232313871384 + }, + "test": { + "mean_centered_corr": 0.0036114430986344814, + "mean_pairwise_disagreement": 0.05469726437468373, + "member_std": 0.05942297354340553 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 14 + ], + "head_selection_time": "0:00:04.742166" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..bf2cd1131 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..329fc00d3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0/report.json @@ -0,0 +1,215 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7862557522292872, + "recall": 0.9783200085532021, + "f1-score": 0.8718353174078054, + "support": 84179.0 + }, + "1": { + "precision": 0.6031746031746031, + "recall": 0.11024560845719736, + "f1-score": 0.18641846712140048, + "support": 25162.0 + }, + "accuracy": 0.7785551622904492, + "macro avg": { + "precision": 0.6947151777019451, + "recall": 0.5442828085051997, + "f1-score": 0.529126892264603, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441243662668947, + "recall": 0.7785551622904492, + "f1-score": 0.7141043767093801, + "support": 109341.0 + }, + "cross-entropy": 0.4922807370219411, + "roc-auc": 0.6989988975549913, + "score": 0.6989988975549913 + }, + "val": { + "0": { + "precision": 0.6750500272876114, + "recall": 0.9565021265627014, + "f1-score": 0.7915000266624007, + "support": 15518.0 + }, + "1": { + "precision": 0.7030356357237132, + "recall": 0.1827747912615807, + "f1-score": 0.29012345679012347, + "support": 8743.0 + }, + "accuracy": 0.6776719838423808, + "macro avg": { + "precision": 0.6890428315056623, + "recall": 0.569638458912141, + "f1-score": 0.5408117417262621, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6851352741676592, + "recall": 0.6776719838423808, + "f1-score": 0.6108176413364323, + "support": 24261.0 + }, + "cross-entropy": 0.6872792555776136, + "roc-auc": 0.6227054517511603, + "score": 0.6227054517511603 + }, + "test": { + "0": { + "precision": 0.656363924552227, + "recall": 0.9788441082614349, + "f1-score": 0.7858057782627259, + "support": 16922.0 + }, + "1": { + "precision": 0.7063166529942576, + "recall": 0.09031784328123361, + "f1-score": 0.16015625, + "support": 9533.0 + }, + "accuracy": 0.6586656586656586, + "macro avg": { + "precision": 0.6813402887732423, + "recall": 0.5345809757713342, + "f1-score": 0.47298101413136295, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6743642783695727, + "recall": 0.6586656586656586, + "f1-score": 0.5603543719906198, + "support": 26455.0 + }, + "cross-entropy": 0.756433532699498, + "roc-auc": 0.591151981311678, + "score": 0.591151981311678 + } + }, + "time": "0:01:00.962097", + "diversity": { + "train": { + "mean_centered_corr": -0.013087093830108643, + "mean_pairwise_disagreement": 0.025822423398123243, + "member_std": 0.02709772437810898 + }, + "val": { + "mean_centered_corr": -0.004288136959075928, + "mean_pairwise_disagreement": 0.08692423855623854, + "member_std": 0.060995232313871384 + }, + "test": { + "mean_centered_corr": 0.0036114430986344814, + "mean_pairwise_disagreement": 0.05469726437468373, + "member_std": 0.05942297354340553 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d0ce06b82 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,227 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7829606930730526, + "recall": 0.9834281709214887, + "f1-score": 0.8718189046448073, + "support": 84179.0 + }, + "1": { + "precision": 0.6134663341645885, + "recall": 0.08798982592798664, + "f1-score": 0.15390497375829829, + "support": 25162.0 + }, + "accuracy": 0.7773662212710694, + "macro avg": { + "precision": 0.6982135136188206, + "recall": 0.5357089984247376, + "f1-score": 0.5128619392015528, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439559550621073, + "recall": 0.7773662212710694, + "f1-score": 0.7066096022882681, + "support": 109341.0 + }, + "cross-entropy": 0.4977984361309486, + "roc-auc": 0.6854164410903827, + "score": 0.6854164410903827 + }, + "val": { + "0": { + "precision": 0.6785484611851171, + "recall": 0.9519267946900374, + "f1-score": 0.7923192447972538, + "support": 15518.0 + }, + "1": { + "precision": 0.7005218787635488, + "recall": 0.19958824202218917, + "f1-score": 0.31066405554566495, + "support": 8743.0 + }, + "accuracy": 0.6808045834879024, + "macro avg": { + "precision": 0.689535169974333, + "recall": 0.5757575183561132, + "f1-score": 0.5514916501714594, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6864670791270086, + "recall": 0.6808045834879024, + "f1-score": 0.6187439049668, + "support": 24261.0 + }, + "cross-entropy": 0.6384330519984234, + "roc-auc": 0.6413971749638401, + "score": 0.6413971749638401 + }, + "test": { + "0": { + "precision": 0.659637469489016, + "recall": 0.9741756293582319, + "f1-score": 0.7866294467110443, + "support": 16922.0 + }, + "1": { + "precision": 0.7015027322404371, + "recall": 0.1077310395468373, + "f1-score": 0.18677821223970173, + "support": 9533.0 + }, + "accuracy": 0.661954261954262, + "macro avg": { + "precision": 0.6805701008647266, + "recall": 0.5409533344525346, + "f1-score": 0.486703829475373, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6747235231578611, + "recall": 0.661954261954262, + "f1-score": 0.5704743978274568, + "support": 26455.0 + }, + "cross-entropy": 0.6791357747346614, + "roc-auc": 0.6040305589800323, + "score": 0.6040305589800323 + } + }, + "time": "0:01:00.962097", + "diversity": { + "train": { + "mean_centered_corr": -0.013087093830108643, + "mean_pairwise_disagreement": 0.025822423398123243, + "member_std": 0.02709772437810898 + }, + "val": { + "mean_centered_corr": -0.004288136959075928, + "mean_pairwise_disagreement": 0.08692423855623854, + "member_std": 0.060995232313871384 + }, + "test": { + "mean_centered_corr": 0.0036114430986344814, + "mean_pairwise_disagreement": 0.05469726437468373, + "member_std": 0.05942297354340553 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 14, + 0, + 25, + 25, + 0, + 25, + 0, + 25, + 0 + ], + "head_selection_time": "0:00:48.379804" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..2d7cfcb35 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,262 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9509438009438009, + "recall": 0.9745069269279171, + "f1-score": 0.9625811844722908, + "support": 181971.0 + }, + "1": { + "precision": 0.8774048625792812, + "recall": 0.7839854542019883, + "f1-score": 0.8280686877252491, + "support": 42349.0 + }, + "accuracy": 0.9385386947218259, + "macro avg": { + "precision": 0.9141743317615411, + "recall": 0.8792461905649527, + "f1-score": 0.89532493609877, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9370605069851746, + "recall": 0.9385386947218259, + "f1-score": 0.9371867937592895, + "support": 224320.0 + }, + "cross-entropy": 0.14097183098455576, + "roc-auc": 0.9772540623630491, + "score": 0.9772540623630491 + }, + "val": { + "0": { + "precision": 0.9410365039753932, + "recall": 0.9685801326085658, + "f1-score": 0.9546096785588132, + "support": 16741.0 + }, + "1": { + "precision": 0.8190574475404196, + "recall": 0.7009125699146306, + "f1-score": 0.7553934010152284, + "support": 3397.0 + }, + "accuracy": 0.923428344423478, + "macro avg": { + "precision": 0.8800469757579064, + "recall": 0.8347463512615982, + "f1-score": 0.8550015397870208, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9204603367934683, + "recall": 0.923428344423478, + "f1-score": 0.9210046683882125, + "support": 20138.0 + }, + "cross-entropy": 0.181709302619563, + "roc-auc": 0.9554763470552776, + "score": 0.9554763470552776 + }, + "test": { + "0": { + "precision": 0.9398939744642724, + "recall": 0.9574807940975127, + "f1-score": 0.9486058779201205, + "support": 13147.0 + }, + "1": { + "precision": 0.8073742246726395, + "recall": 0.7442820838627701, + "f1-score": 0.7745454545454545, + "support": 3148.0 + }, + "accuracy": 0.9162933415158024, + "macro avg": { + "precision": 0.873634099568456, + "recall": 0.8508814389801413, + "f1-score": 0.8615756662327876, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9142927365174138, + "recall": 0.9162933415158024, + "f1-score": 0.914979476399197, + "support": 16295.0 + }, + "cross-entropy": 0.18955508005758365, + "roc-auc": 0.959416932798502, + "score": 0.959416932798502 + } + }, + "time": "0:02:48.131677", + "diversity": { + "train": { + "mean_centered_corr": 0.05291927233338356, + "mean_pairwise_disagreement": 0.03400679940235148, + "member_std": 0.03158120810985565 + }, + "val": { + "mean_centered_corr": 0.06288693100214005, + "mean_pairwise_disagreement": 0.03244768356405319, + "member_std": 0.0349886491894722 + }, + "test": { + "mean_centered_corr": 0.07999585568904877, + "mean_pairwise_disagreement": 0.041502439893495925, + "member_std": 0.0369756743311882 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.912933" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..d85b237a3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0.toml @@ -0,0 +1,34 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..3e79b9f86 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0/report.json @@ -0,0 +1,258 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9482836172363248, + "recall": 0.984772298882789, + "f1-score": 0.966183574879227, + "support": 181971.0 + }, + "1": { + "precision": 0.9216057939853453, + "recall": 0.769227136414083, + "f1-score": 0.8385502471169687, + "support": 42349.0 + }, + "accuracy": 0.9440798858773182, + "macro avg": { + "precision": 0.9349447056108351, + "recall": 0.8769997176484361, + "f1-score": 0.9023669109980978, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.943247155320955, + "recall": 0.9440798858773182, + "f1-score": 0.9420878910462924, + "support": 224320.0 + }, + "cross-entropy": 0.13271224543526663, + "roc-auc": 0.9813876825616359, + "score": 0.9813876825616359 + }, + "val": { + "0": { + "precision": 0.9351403678606002, + "recall": 0.9809449853652709, + "f1-score": 0.9574951897848522, + "support": 16741.0 + }, + "1": { + "precision": 0.8762126503686457, + "recall": 0.6647041507212246, + "f1-score": 0.7559424171409441, + "support": 3397.0 + }, + "accuracy": 0.9275995630151952, + "macro avg": { + "precision": 0.9056765091146229, + "recall": 0.8228245680432478, + "f1-score": 0.8567188034628981, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9252000830100605, + "recall": 0.9275995630151952, + "f1-score": 0.9234960454472141, + "support": 20138.0 + }, + "cross-entropy": 0.17351272311805546, + "roc-auc": 0.9583121450834431, + "score": 0.9583121450834431 + }, + "test": { + "0": { + "precision": 0.9321849105974706, + "recall": 0.975507720392485, + "f1-score": 0.9533543950938487, + "support": 13147.0 + }, + "1": { + "precision": 0.8730784391013008, + "recall": 0.7036213468869124, + "f1-score": 0.7792436235708003, + "support": 3148.0 + }, + "accuracy": 0.9229825099723842, + "macro avg": { + "precision": 0.9026316748493857, + "recall": 0.8395645336396986, + "f1-score": 0.8662990093323245, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9207662439960627, + "recall": 0.9229825099723842, + "f1-score": 0.9197182669100772, + "support": 16295.0 + }, + "cross-entropy": 0.17696141048282132, + "roc-auc": 0.9626260391126089, + "score": 0.9626260391126089 + } + }, + "time": "0:02:48.131677", + "diversity": { + "train": { + "mean_centered_corr": 0.05291927233338356, + "mean_pairwise_disagreement": 0.03400679940235148, + "member_std": 0.03158120810985565 + }, + "val": { + "mean_centered_corr": 0.06288693100214005, + "mean_pairwise_disagreement": 0.03244768356405319, + "member_std": 0.0349886491894722 + }, + "test": { + "mean_centered_corr": 0.07999585568904877, + "mean_pairwise_disagreement": 0.041502439893495925, + "member_std": 0.0369756743311882 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d5d5b7b36 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9453926712602988, + "recall": 0.9862120887394146, + "f1-score": 0.9653710741556594, + "support": 181971.0 + }, + "1": { + "precision": 0.9272584947234142, + "recall": 0.7552244444969185, + "f1-score": 0.8324462201168646, + "support": 42349.0 + }, + "accuracy": 0.9426043152639088, + "macro avg": { + "precision": 0.9363255829918564, + "recall": 0.8707182666181665, + "f1-score": 0.898908647136262, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9419691502092978, + "recall": 0.9426043152639088, + "f1-score": 0.9402764118710262, + "support": 224320.0 + }, + "cross-entropy": 0.13432360872538882, + "roc-auc": 0.9807020207732833, + "score": 0.9807020207732833 + }, + "val": { + "0": { + "precision": 0.9332728098048116, + "recall": 0.9824980586583836, + "f1-score": 0.9572530190600902, + "support": 16741.0 + }, + "1": { + "precision": 0.8834526650755767, + "recall": 0.6538121872240212, + "f1-score": 0.7514802909829132, + "support": 3397.0 + }, + "accuracy": 0.927053332009137, + "macro avg": { + "precision": 0.9083627374401941, + "recall": 0.8181551229412024, + "f1-score": 0.8543666550215017, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9248688455757318, + "recall": 0.927053332009137, + "f1-score": 0.9225420270411127, + "support": 20138.0 + }, + "cross-entropy": 0.17301902153958362, + "roc-auc": 0.9589474980445032, + "score": 0.9589474980445032 + }, + "test": { + "0": { + "precision": 0.9310144506571781, + "recall": 0.9752034684718948, + "f1-score": 0.952596775391931, + "support": 13147.0 + }, + "1": { + "precision": 0.8708399366085579, + "recall": 0.6982210927573063, + "f1-score": 0.7750352609308886, + "support": 3148.0 + }, + "accuracy": 0.921693771095428, + "macro avg": { + "precision": 0.900927193632868, + "recall": 0.8367122806146006, + "f1-score": 0.8638160181614098, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9193894509502093, + "recall": 0.921693771095428, + "f1-score": 0.9182940047553332, + "support": 16295.0 + }, + "cross-entropy": 0.17739591453615167, + "roc-auc": 0.9625170670540112, + "score": 0.9625170670540112 + } + }, + "time": "0:02:48.131677", + "diversity": { + "train": { + "mean_centered_corr": 0.05291927233338356, + "mean_pairwise_disagreement": 0.03400679940235148, + "member_std": 0.03158120810985565 + }, + "val": { + "mean_centered_corr": 0.06288693100214005, + "mean_pairwise_disagreement": 0.03244768356405319, + "member_std": 0.0349886491894722 + }, + "test": { + "mean_centered_corr": 0.07999585568904877, + "mean_pairwise_disagreement": 0.041502439893495925, + "member_std": 0.0369756743311882 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 6, + 9, + 15, + 28, + 3, + 23, + 26, + 2, + 20, + 13, + 6, + 15, + 5 + ], + "head_selection_time": "0:00:13.989889" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ff85a29bb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,190 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22658863410177663, + "mae": 0.12739618122577667, + "r2": 0.6719876527786255, + "score": -0.22658863410177663 + }, + "val": { + "rmse": 0.23955454428788905, + "mae": 0.13915589451789856, + "r2": 0.6166812181472778, + "score": -0.23955454428788905 + }, + "test": { + "rmse": 0.26588264760298186, + "mae": 0.1863766759634018, + "r2": 0.5169194340705872, + "score": -0.26588264760298186 + } + }, + "time": "0:00:48.779291", + "diversity": { + "train": { + "mean_centered_corr": -0.018623020499944687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05047355964779854 + }, + "val": { + "mean_centered_corr": -0.01870153285562992, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06864207983016968 + }, + "test": { + "mean_centered_corr": -0.019026711583137512, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09529124945402145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015297" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..a68e82651 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0.toml @@ -0,0 +1,34 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..cfb76d49d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0/report.json @@ -0,0 +1,186 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22224988999004666, + "mae": 0.11853151023387909, + "r2": 0.6844290494918823, + "score": -0.22224988999004666 + }, + "val": { + "rmse": 0.23452539387100468, + "mae": 0.12745676934719086, + "r2": 0.6326068639755249, + "score": -0.23452539387100468 + }, + "test": { + "rmse": 0.23846659721529556, + "mae": 0.15488465130329132, + "r2": 0.6114072799682617, + "score": -0.23846659721529556 + } + }, + "time": "0:00:48.779291", + "diversity": { + "train": { + "mean_centered_corr": -0.018623020499944687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05047355964779854 + }, + "val": { + "mean_centered_corr": -0.01870153285562992, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06864207983016968 + }, + "test": { + "mean_centered_corr": -0.019026711583137512, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09529124945402145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..e6d5a6174 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22302489965783162, + "mae": 0.11715596914291382, + "r2": 0.6822243928909302, + "score": -0.22302489965783162 + }, + "val": { + "rmse": 0.23257941028161666, + "mae": 0.12573255598545074, + "r2": 0.6386785507202148, + "score": -0.23257941028161666 + }, + "test": { + "rmse": 0.23549883338311248, + "mae": 0.14942243695259094, + "r2": 0.6210193037986755, + "score": -0.23549883338311248 + } + }, + "time": "0:00:48.779291", + "diversity": { + "train": { + "mean_centered_corr": -0.018623020499944687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05047355964779854 + }, + "val": { + "mean_centered_corr": -0.01870153285562992, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06864207983016968 + }, + "test": { + "mean_centered_corr": -0.019026711583137512, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09529124945402145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 19, + 21, + 12, + 16, + 18, + 8, + 22, + 19, + 29 + ], + "head_selection_time": "0:00:00.160128" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0.toml deleted file mode 100644 index eb4b802dd..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0.toml +++ /dev/null @@ -1,35 +0,0 @@ -dataset = "cooking-time" -variant = "all_four_combined" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0/report.json deleted file mode 100644 index 1533a55d0..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0/report.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 5, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_four_combined" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.597904784280686, - "train_loss": 0.5734070920729423, - "val_score": -2.6454041701159015 - }, - { - "epoch": 1, - "test_score": -2.5479064912091176, - "train_loss": 0.519261709741644, - "val_score": -2.5936452086088506 - }, - { - "epoch": 2, - "test_score": -2.568095726724589, - "train_loss": 0.5110039501576811, - "val_score": -2.6121638579153807 - }, - { - "epoch": 3, - "test_score": -2.5576997782221387, - "train_loss": 0.5093689668285954, - "val_score": -2.6009487768495023 - }, - { - "epoch": 4, - "test_score": -2.5850651586352034, - "train_loss": 0.5089051253623791, - "val_score": -2.6281073207918295 - }, - { - "epoch": 5, - "test_score": -2.5511833212856643, - "train_loss": 0.5059712389567951, - "val_score": -2.593261672781138 - }, - { - "epoch": 6, - "test_score": -2.551401433865984, - "train_loss": 0.5055853670781797, - "val_score": -2.5934982177598815 - }, - { - "epoch": 7, - "test_score": -2.5609792523241888, - "train_loss": 0.5017106358532433, - "val_score": -2.5997025539743244 - }, - { - "epoch": 8, - "test_score": -2.566694687611926, - "train_loss": 0.5020474761992961, - "val_score": -2.6070956936954204 - }, - { - "epoch": 9, - "test_score": -2.5532860134344815, - "train_loss": 0.5024598673120275, - "val_score": -2.5942565239541864 - }, - { - "epoch": 10, - "test_score": -2.579700982412669, - "train_loss": 0.499374769829415, - "val_score": -2.619462575902537 - }, - { - "epoch": 11, - "test_score": -2.557298172102745, - "train_loss": 0.4954140352773237, - "val_score": -2.597185364686641 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.5511833212856643, - "score": -2.5511833212856643 - }, - "val": { - "rmse": 2.593261672781138, - "score": -2.593261672781138 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 50.65602159500122, - "variant": "all_four_combined" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1.toml deleted file mode 100644 index 7dbf8b11c..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1.toml +++ /dev/null @@ -1,35 +0,0 @@ -dataset = "cooking-time" -variant = "all_four_combined" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1/report.json deleted file mode 100644 index 3402f791d..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1/report.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_four_combined" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.537508726809674, - "train_loss": 0.5776582470348289, - "val_score": -2.5844468721407705 - }, - { - "epoch": 1, - "test_score": -2.563174833433257, - "train_loss": 0.5161436847201338, - "val_score": -2.607712907300778 - }, - { - "epoch": 2, - "test_score": -2.5679362249447126, - "train_loss": 0.5124912057910953, - "val_score": -2.6130100902153788 - }, - { - "epoch": 3, - "test_score": -2.5667008183011366, - "train_loss": 0.509442943978954, - "val_score": -2.609481352506116 - }, - { - "epoch": 4, - "test_score": -2.581905206094061, - "train_loss": 0.5058386825226449, - "val_score": -2.6246502052852225 - }, - { - "epoch": 5, - "test_score": -2.558270476207951, - "train_loss": 0.5066947660467647, - "val_score": -2.599870010941221 - }, - { - "epoch": 6, - "test_score": -2.5907350965419256, - "train_loss": 0.504907029974568, - "val_score": -2.6330101776621766 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.537508726809674, - "score": -2.537508726809674 - }, - "val": { - "rmse": 2.5844468721407705, - "score": -2.5844468721407705 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 30.867638111114502, - "variant": "all_four_combined" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2.toml deleted file mode 100644 index 434c87c95..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2.toml +++ /dev/null @@ -1,35 +0,0 @@ -dataset = "cooking-time" -variant = "all_four_combined" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2/report.json deleted file mode 100644 index 9b3be73a4..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2/report.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 2, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_four_combined" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.553360807438672, - "train_loss": 0.5795920593244536, - "val_score": -2.6008898349415337 - }, - { - "epoch": 1, - "test_score": -2.5549795001377738, - "train_loss": 0.5163620345227353, - "val_score": -2.600778914198739 - }, - { - "epoch": 2, - "test_score": -2.543946905739664, - "train_loss": 0.5128198091511253, - "val_score": -2.589426873028416 - }, - { - "epoch": 3, - "test_score": -2.55344026783191, - "train_loss": 0.5110959424091889, - "val_score": -2.5967557106208683 - }, - { - "epoch": 4, - "test_score": -2.550100612525452, - "train_loss": 0.5060882082393577, - "val_score": -2.5920808316036634 - }, - { - "epoch": 5, - "test_score": -2.584788271646388, - "train_loss": 0.5051669679783486, - "val_score": -2.624746855382455 - }, - { - "epoch": 6, - "test_score": -2.5773501994782806, - "train_loss": 0.5029631283906129, - "val_score": -2.6191585577138237 - }, - { - "epoch": 7, - "test_score": -2.5664566940854403, - "train_loss": 0.5020166579130534, - "val_score": -2.6072144843786274 - }, - { - "epoch": 8, - "test_score": -2.557857774486442, - "train_loss": 0.5009360981954111, - "val_score": -2.5982997470109916 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.543946905739664, - "score": -2.543946905739664 - }, - "val": { - "rmse": 2.589426873028416, - "score": -2.589426873028416 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_four_combined-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 38.606435775756836, - "variant": "all_four_combined" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0.toml deleted file mode 100644 index d0a7f27f4..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0.toml +++ /dev/null @@ -1,34 +0,0 @@ -dataset = "cooking-time" -variant = "all_minus_cf_fisd" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0/report.json deleted file mode 100644 index 8fd9c751e..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0/report.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 2, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_minus_cf_fisd" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.597582914995142, - "train_loss": 0.5761679290114222, - "val_score": -2.6451580257118 - }, - { - "epoch": 1, - "test_score": -2.556929604770859, - "train_loss": 0.5164911712075139, - "val_score": -2.600416784626239 - }, - { - "epoch": 2, - "test_score": -2.546692730518933, - "train_loss": 0.5091461243930163, - "val_score": -2.5875247567139654 - }, - { - "epoch": 3, - "test_score": -2.5665785733087403, - "train_loss": 0.5083916611499615, - "val_score": -2.60850327601345 - }, - { - "epoch": 4, - "test_score": -2.592980420028019, - "train_loss": 0.5077804998234585, - "val_score": -2.634938997145239 - }, - { - "epoch": 5, - "test_score": -2.5491212639740826, - "train_loss": 0.5051470538517376, - "val_score": -2.591631196281184 - }, - { - "epoch": 6, - "test_score": -2.5484263370159503, - "train_loss": 0.5052224558752936, - "val_score": -2.5904392115991604 - }, - { - "epoch": 7, - "test_score": -2.5579466023184496, - "train_loss": 0.5017991039130065, - "val_score": -2.595014516683264 - }, - { - "epoch": 8, - "test_score": -2.5690880731651538, - "train_loss": 0.5029040357551059, - "val_score": -2.6090379143179177 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.546692730518933, - "score": -2.546692730518933 - }, - "val": { - "rmse": 2.5875247567139654, - "score": -2.5875247567139654 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 38.443965435028076, - "variant": "all_minus_cf_fisd" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1.toml deleted file mode 100644 index ebfe27a78..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1.toml +++ /dev/null @@ -1,34 +0,0 @@ -dataset = "cooking-time" -variant = "all_minus_cf_fisd" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1/report.json deleted file mode 100644 index 0130e6f0f..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1/report.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_minus_cf_fisd" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.5418282337637406, - "train_loss": 0.5805648884794734, - "val_score": -2.5864378108053976 - }, - { - "epoch": 1, - "test_score": -2.569724807106596, - "train_loss": 0.5154956329513241, - "val_score": -2.6111584810149067 - }, - { - "epoch": 2, - "test_score": -2.5603445338327986, - "train_loss": 0.511341643763018, - "val_score": -2.605412930143715 - }, - { - "epoch": 3, - "test_score": -2.5649989898858117, - "train_loss": 0.5074708193272084, - "val_score": -2.6060717112349616 - }, - { - "epoch": 4, - "test_score": -2.584029678097529, - "train_loss": 0.5050664694459589, - "val_score": -2.6266244675623907 - }, - { - "epoch": 5, - "test_score": -2.557123265415895, - "train_loss": 0.506016700385927, - "val_score": -2.599371184822983 - }, - { - "epoch": 6, - "test_score": -2.5861473344104584, - "train_loss": 0.5048232132250124, - "val_score": -2.628480692594635 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.5418282337637406, - "score": -2.5418282337637406 - }, - "val": { - "rmse": 2.5864378108053976, - "score": -2.5864378108053976 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 30.901761054992676, - "variant": "all_minus_cf_fisd" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2.toml deleted file mode 100644 index 1c796107d..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2.toml +++ /dev/null @@ -1,34 +0,0 @@ -dataset = "cooking-time" -variant = "all_minus_cf_fisd" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2/report.json deleted file mode 100644 index e41d048d7..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2/report.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 1, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_minus_cf_fisd" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.5508004110414038, - "train_loss": 0.5819202207767211, - "val_score": -2.5966282696347784 - }, - { - "epoch": 1, - "test_score": -2.547009985219361, - "train_loss": 0.5161475149897842, - "val_score": -2.5906049665497832 - }, - { - "epoch": 2, - "test_score": -2.5508009718502036, - "train_loss": 0.5122214724888673, - "val_score": -2.5949883319981977 - }, - { - "epoch": 3, - "test_score": -2.564394366948711, - "train_loss": 0.5101304043520678, - "val_score": -2.607212929801097 - }, - { - "epoch": 4, - "test_score": -2.5620153015239624, - "train_loss": 0.505169359950332, - "val_score": -2.6036783498061484 - }, - { - "epoch": 5, - "test_score": -2.583565906597254, - "train_loss": 0.5044403819887487, - "val_score": -2.622261708127521 - }, - { - "epoch": 6, - "test_score": -2.5759304069394546, - "train_loss": 0.5026040581969528, - "val_score": -2.617307381232943 - }, - { - "epoch": 7, - "test_score": -2.568862738072886, - "train_loss": 0.5019908485648868, - "val_score": -2.607877838614469 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.547009985219361, - "score": -2.547009985219361 - }, - "val": { - "rmse": 2.5906049665497832, - "score": -2.5906049665497832 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_cf_fisd-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 34.84113788604736, - "variant": "all_minus_cf_fisd" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0.toml deleted file mode 100644 index 7beafa65c..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "cooking-time" -variant = "all_minus_esam" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0/report.json deleted file mode 100644 index 581caaef2..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0/report.json +++ /dev/null @@ -1,182 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 13, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_minus_esam" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.5747772676005756, - "train_loss": 0.5732710495725408, - "val_score": -2.6218253420831923 - }, - { - "epoch": 1, - "test_score": -2.5564936514763854, - "train_loss": 0.5177499753934843, - "val_score": -2.6013183479183777 - }, - { - "epoch": 2, - "test_score": -2.5814181478565716, - "train_loss": 0.5109633602000572, - "val_score": -2.62519908331745 - }, - { - "epoch": 3, - "test_score": -2.574927827138447, - "train_loss": 0.5100945451775113, - "val_score": -2.6186524803194304 - }, - { - "epoch": 4, - "test_score": -2.5742446828644434, - "train_loss": 0.5061127572983235, - "val_score": -2.6172427954454816 - }, - { - "epoch": 5, - "test_score": -2.55020008798114, - "train_loss": 0.5048669292046143, - "val_score": -2.5901328913644526 - }, - { - "epoch": 6, - "test_score": -2.572474164916452, - "train_loss": 0.5023283319430308, - "val_score": -2.614022233998335 - }, - { - "epoch": 7, - "test_score": -2.5462834894134376, - "train_loss": 0.5025209985874795, - "val_score": -2.5865856637593847 - }, - { - "epoch": 8, - "test_score": -2.591484091195984, - "train_loss": 0.5010600809578423, - "val_score": -2.6324075902530604 - }, - { - "epoch": 9, - "test_score": -2.5581126918352166, - "train_loss": 0.5000964628683554, - "val_score": -2.598999593913042 - }, - { - "epoch": 10, - "test_score": -2.581940111176364, - "train_loss": 0.4989573072205793, - "val_score": -2.621668745437383 - }, - { - "epoch": 11, - "test_score": -2.554036842104361, - "train_loss": 0.49763295591414514, - "val_score": -2.594907571252727 - }, - { - "epoch": 12, - "test_score": -2.575861729251762, - "train_loss": 0.4963759712807767, - "val_score": -2.614391142530623 - }, - { - "epoch": 13, - "test_score": -2.5437589902976687, - "train_loss": 0.49609845984089485, - "val_score": -2.5833819805199907 - }, - { - "epoch": 14, - "test_score": -2.545113732695165, - "train_loss": 0.49491846615129764, - "val_score": -2.584010947992617 - }, - { - "epoch": 15, - "test_score": -2.5663227317377366, - "train_loss": 0.49232708306999895, - "val_score": -2.6043675968219655 - }, - { - "epoch": 16, - "test_score": -2.5473062349102906, - "train_loss": 0.4936659234064119, - "val_score": -2.585324125352541 - }, - { - "epoch": 17, - "test_score": -2.5794402495575657, - "train_loss": 0.4931852578579842, - "val_score": -2.618063162453346 - }, - { - "epoch": 18, - "test_score": -2.563614486616978, - "train_loss": 0.49165967402157484, - "val_score": -2.6031335434424254 - }, - { - "epoch": 19, - "test_score": -2.5898691581757034, - "train_loss": 0.4904036406461183, - "val_score": -2.6268180727366444 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.5437589902976687, - "score": -2.5437589902976687 - }, - "val": { - "rmse": 2.5833819805199907, - "score": -2.5833819805199907 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 58.42725872993469, - "variant": "all_minus_esam" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1.toml deleted file mode 100644 index d712fef7c..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "cooking-time" -variant = "all_minus_esam" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1/report.json deleted file mode 100644 index dc4288986..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1/report.json +++ /dev/null @@ -1,134 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 5, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_minus_esam" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.546896343492815, - "train_loss": 0.5771154447181804, - "val_score": -2.5931625620194123 - }, - { - "epoch": 1, - "test_score": -2.563574588899563, - "train_loss": 0.5195186444230981, - "val_score": -2.608936935475554 - }, - { - "epoch": 2, - "test_score": -2.576534080926977, - "train_loss": 0.5107925359730248, - "val_score": -2.6210837442175823 - }, - { - "epoch": 3, - "test_score": -2.5532837723818225, - "train_loss": 0.5060298748918481, - "val_score": -2.5953448803998618 - }, - { - "epoch": 4, - "test_score": -2.566363144139515, - "train_loss": 0.5067531010052105, - "val_score": -2.6086219112022055 - }, - { - "epoch": 5, - "test_score": -2.538235385864352, - "train_loss": 0.5049025134460347, - "val_score": -2.5802569273242226 - }, - { - "epoch": 6, - "test_score": -2.5821721533391186, - "train_loss": 0.5031720842864063, - "val_score": -2.622583639522115 - }, - { - "epoch": 7, - "test_score": -2.573924856818587, - "train_loss": 0.5019908636002928, - "val_score": -2.6141672499897917 - }, - { - "epoch": 8, - "test_score": -2.5526273394139105, - "train_loss": 0.5017939341497851, - "val_score": -2.5910970149720978 - }, - { - "epoch": 9, - "test_score": -2.567319849634468, - "train_loss": 0.5008838630474366, - "val_score": -2.6068355058135366 - }, - { - "epoch": 10, - "test_score": -2.591484275197554, - "train_loss": 0.49883278210957843, - "val_score": -2.6313516882594596 - }, - { - "epoch": 11, - "test_score": -2.5877905714738745, - "train_loss": 0.49841721772073627, - "val_score": -2.6277731824390678 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.538235385864352, - "score": -2.538235385864352 - }, - "val": { - "rmse": 2.5802569273242226, - "score": -2.5802569273242226 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 37.65957021713257, - "variant": "all_minus_esam" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2.toml deleted file mode 100644 index 9c131ba6b..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "cooking-time" -variant = "all_minus_esam" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2/report.json deleted file mode 100644 index fc4a1e67b..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2/report.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 7, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_minus_esam" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.5613993630298912, - "train_loss": 0.5778566953298208, - "val_score": -2.6107928598405943 - }, - { - "epoch": 1, - "test_score": -2.550575797203022, - "train_loss": 0.5199141429888235, - "val_score": -2.5975772243324835 - }, - { - "epoch": 2, - "test_score": -2.5513964812165146, - "train_loss": 0.511967933661229, - "val_score": -2.5944791021852587 - }, - { - "epoch": 3, - "test_score": -2.5550999672998134, - "train_loss": 0.5085972109893421, - "val_score": -2.598609416731633 - }, - { - "epoch": 4, - "test_score": -2.5458052559244453, - "train_loss": 0.5063928671785303, - "val_score": -2.5876654530576864 - }, - { - "epoch": 5, - "test_score": -2.587354749942648, - "train_loss": 0.5060564223173503, - "val_score": -2.629049084709761 - }, - { - "epoch": 6, - "test_score": -2.566285756155162, - "train_loss": 0.5039662869126947, - "val_score": -2.607049236744262 - }, - { - "epoch": 7, - "test_score": -2.5476520499692263, - "train_loss": 0.5022442496574677, - "val_score": -2.586853602657961 - }, - { - "epoch": 8, - "test_score": -2.5592359828805464, - "train_loss": 0.5007378045503084, - "val_score": -2.599643675522678 - }, - { - "epoch": 9, - "test_score": -2.5641869365431083, - "train_loss": 0.49786975281732576, - "val_score": -2.6038341055557614 - }, - { - "epoch": 10, - "test_score": -2.580490599126014, - "train_loss": 0.4966755004616471, - "val_score": -2.6210794690079933 - }, - { - "epoch": 11, - "test_score": -2.571333291165615, - "train_loss": 0.49655182189769576, - "val_score": -2.6103619747617812 - }, - { - "epoch": 12, - "test_score": -2.5867859523181176, - "train_loss": 0.4945516618522438, - "val_score": -2.6269796263765466 - }, - { - "epoch": 13, - "test_score": -2.5806335267258453, - "train_loss": 0.49264940216734604, - "val_score": -2.620915641361488 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.5476520499692263, - "score": -2.5476520499692263 - }, - "val": { - "rmse": 2.586853602657961, - "score": -2.586853602657961 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_esam-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 43.24603581428528, - "variant": "all_minus_esam" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0.toml deleted file mode 100644 index 73a24206b..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "cooking-time" -variant = "all_minus_mfb" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0/report.json deleted file mode 100644 index 1c3f751de..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0/report.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 3, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_minus_mfb" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.603120171403929, - "train_loss": 0.5561879529609336, - "val_score": -2.642832628894953 - }, - { - "epoch": 1, - "test_score": -2.588518681158689, - "train_loss": 0.4951903846886781, - "val_score": -2.626524527943006 - }, - { - "epoch": 2, - "test_score": -2.572048725550596, - "train_loss": 0.4900382980033084, - "val_score": -2.608899467218641 - }, - { - "epoch": 3, - "test_score": -2.572318549397406, - "train_loss": 0.4874864126647915, - "val_score": -2.60825858532094 - }, - { - "epoch": 4, - "test_score": -2.5802546172956813, - "train_loss": 0.4863069078943751, - "val_score": -2.6160648500237964 - }, - { - "epoch": 5, - "test_score": -2.5864119080151102, - "train_loss": 0.4848573691136128, - "val_score": -2.6218528046424394 - }, - { - "epoch": 6, - "test_score": -2.5831701678358927, - "train_loss": 0.48349400683566257, - "val_score": -2.617953424684935 - }, - { - "epoch": 7, - "test_score": -2.5753206664480017, - "train_loss": 0.48293743423513463, - "val_score": -2.6101944601761153 - }, - { - "epoch": 8, - "test_score": -2.5976935133018086, - "train_loss": 0.4819742309617567, - "val_score": -2.6332332829490097 - }, - { - "epoch": 9, - "test_score": -2.5914011971600104, - "train_loss": 0.4817004324616613, - "val_score": -2.623775287304586 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.572318549397406, - "score": -2.572318549397406 - }, - "val": { - "rmse": 2.60825858532094, - "score": -2.60825858532094 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 42.692240953445435, - "variant": "all_minus_mfb" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1.toml deleted file mode 100644 index 2cbd4e61c..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "cooking-time" -variant = "all_minus_mfb" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1/report.json deleted file mode 100644 index 9230108cd..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1/report.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 3, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_minus_mfb" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.561574350040879, - "train_loss": 0.5604447545232, - "val_score": -2.6029467866304032 - }, - { - "epoch": 1, - "test_score": -2.574204301576486, - "train_loss": 0.4954050814246272, - "val_score": -2.6123855492741956 - }, - { - "epoch": 2, - "test_score": -2.5578780010321176, - "train_loss": 0.49020274399637104, - "val_score": -2.5961486571822516 - }, - { - "epoch": 3, - "test_score": -2.5472973432410306, - "train_loss": 0.48780881499384976, - "val_score": -2.5833460797231274 - }, - { - "epoch": 4, - "test_score": -2.593680599157413, - "train_loss": 0.4862094964530017, - "val_score": -2.62832068260243 - }, - { - "epoch": 5, - "test_score": -2.569472340841902, - "train_loss": 0.4844946855897302, - "val_score": -2.605646267495547 - }, - { - "epoch": 6, - "test_score": -2.5842775854642452, - "train_loss": 0.4840661797437582, - "val_score": -2.618392530274335 - }, - { - "epoch": 7, - "test_score": -2.5719627949410135, - "train_loss": 0.48236743233225365, - "val_score": -2.6085260346458674 - }, - { - "epoch": 8, - "test_score": -2.561136580823507, - "train_loss": 0.48235606825029526, - "val_score": -2.5962453581502647 - }, - { - "epoch": 9, - "test_score": -2.57570807712001, - "train_loss": 0.48123758950748957, - "val_score": -2.611934571473405 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.5472973432410306, - "score": -2.5472973432410306 - }, - "val": { - "rmse": 2.5833460797231274, - "score": -2.5833460797231274 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 44.12882924079895, - "variant": "all_minus_mfb" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2.toml deleted file mode 100644 index 990e8d379..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "cooking-time" -variant = "all_minus_mfb" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2/report.json deleted file mode 100644 index f593ca0d5..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2/report.json +++ /dev/null @@ -1,110 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 1, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_minus_mfb" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.603077673510457, - "train_loss": 0.5612956862728875, - "val_score": -2.643770770629382 - }, - { - "epoch": 1, - "test_score": -2.5601664827274426, - "train_loss": 0.4957994515294427, - "val_score": -2.599479505818785 - }, - { - "epoch": 2, - "test_score": -2.5856204112203303, - "train_loss": 0.4903566123129011, - "val_score": -2.6224372706443058 - }, - { - "epoch": 3, - "test_score": -2.5785627513659493, - "train_loss": 0.4880300644818727, - "val_score": -2.6154062159108005 - }, - { - "epoch": 4, - "test_score": -2.5974606547895833, - "train_loss": 0.48578819578832333, - "val_score": -2.63218314688471 - }, - { - "epoch": 5, - "test_score": -2.5809724746294607, - "train_loss": 0.4853922931460647, - "val_score": -2.615697727081015 - }, - { - "epoch": 6, - "test_score": -2.5823524725431946, - "train_loss": 0.4840138941197782, - "val_score": -2.6170042970219027 - }, - { - "epoch": 7, - "test_score": -2.568776700885788, - "train_loss": 0.48290939508257685, - "val_score": -2.6039036020287787 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.5601664827274426, - "score": -2.5601664827274426 - }, - "val": { - "rmse": 2.599479505818785, - "score": -2.599479505818785 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_mfb-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 35.43584752082825, - "variant": "all_minus_mfb" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0.toml deleted file mode 100644 index c0c763e13..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0.toml +++ /dev/null @@ -1,32 +0,0 @@ -dataset = "cooking-time" -variant = "all_minus_rla" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0/report.json deleted file mode 100644 index aafa0dc07..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0/report.json +++ /dev/null @@ -1,139 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 6, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_minus_rla" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.5764028635071594, - "train_loss": 0.573144529048387, - "val_score": -2.6233867047352715 - }, - { - "epoch": 1, - "test_score": -2.5481321824369987, - "train_loss": 0.518380832296234, - "val_score": -2.593426328019743 - }, - { - "epoch": 2, - "test_score": -2.568308596961209, - "train_loss": 0.5127465303416725, - "val_score": -2.6122966559455976 - }, - { - "epoch": 3, - "test_score": -2.56838332480516, - "train_loss": 0.5098949592929702, - "val_score": -2.6127086975845275 - }, - { - "epoch": 4, - "test_score": -2.5867971967751267, - "train_loss": 0.5094581716232471, - "val_score": -2.6301702716956505 - }, - { - "epoch": 5, - "test_score": -2.557593136878615, - "train_loss": 0.5076963931053609, - "val_score": -2.5998900940480447 - }, - { - "epoch": 6, - "test_score": -2.550683292768776, - "train_loss": 0.5074400810508041, - "val_score": -2.5932213118056806 - }, - { - "epoch": 7, - "test_score": -2.5802537856849, - "train_loss": 0.5039221112792557, - "val_score": -2.6213640735928387 - }, - { - "epoch": 8, - "test_score": -2.5604579511458847, - "train_loss": 0.5042530888909692, - "val_score": -2.601104512726605 - }, - { - "epoch": 9, - "test_score": -2.5711370844102874, - "train_loss": 0.5058108028527852, - "val_score": -2.6124406725704215 - }, - { - "epoch": 10, - "test_score": -2.582553089404887, - "train_loss": 0.5030083006566709, - "val_score": -2.6239106779073076 - }, - { - "epoch": 11, - "test_score": -2.575217902583891, - "train_loss": 0.49936754934422606, - "val_score": -2.616333961798112 - }, - { - "epoch": 12, - "test_score": -2.5628126000259095, - "train_loss": 0.5023714462379077, - "val_score": -2.6039989163093247 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.550683292768776, - "score": -2.550683292768776 - }, - "val": { - "rmse": 2.5932213118056806, - "score": -2.5932213118056806 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 49.47507905960083, - "variant": "all_minus_rla" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1.toml deleted file mode 100644 index 05d224989..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1.toml +++ /dev/null @@ -1,32 +0,0 @@ -dataset = "cooking-time" -variant = "all_minus_rla" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1/report.json deleted file mode 100644 index b6f3ebac3..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1/report.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_minus_rla" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.5451569174185407, - "train_loss": 0.5805180867513021, - "val_score": -2.5924488158953323 - }, - { - "epoch": 1, - "test_score": -2.5729383604136413, - "train_loss": 0.5178044803507693, - "val_score": -2.618478940350594 - }, - { - "epoch": 2, - "test_score": -2.573552463652647, - "train_loss": 0.5117641367890813, - "val_score": -2.617134207567258 - }, - { - "epoch": 3, - "test_score": -2.566955414750782, - "train_loss": 0.5101196943102656, - "val_score": -2.6105366022963 - }, - { - "epoch": 4, - "test_score": -2.573111821442705, - "train_loss": 0.5074310834343368, - "val_score": -2.617440100479082 - }, - { - "epoch": 5, - "test_score": -2.5662011190851253, - "train_loss": 0.5082239653613116, - "val_score": -2.6090179016271127 - }, - { - "epoch": 6, - "test_score": -2.5780162441549805, - "train_loss": 0.5056800353634465, - "val_score": -2.6210339876232287 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.5451569174185407, - "score": -2.5451569174185407 - }, - "val": { - "rmse": 2.5924488158953323, - "score": -2.5924488158953323 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 26.603979349136353, - "variant": "all_minus_rla" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2.toml deleted file mode 100644 index e5fdb6826..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2.toml +++ /dev/null @@ -1,32 +0,0 @@ -dataset = "cooking-time" -variant = "all_minus_rla" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2/report.json deleted file mode 100644 index 7e716b47b..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2/report.json +++ /dev/null @@ -1,157 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 9, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_minus_rla" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.564628926255888, - "train_loss": 0.5797603635100631, - "val_score": -2.6127031311205617 - }, - { - "epoch": 1, - "test_score": -2.567705031902881, - "train_loss": 0.5178446670373281, - "val_score": -2.6129844508389115 - }, - { - "epoch": 2, - "test_score": -2.562137019803166, - "train_loss": 0.5133768392575754, - "val_score": -2.607213752812846 - }, - { - "epoch": 3, - "test_score": -2.584532756672374, - "train_loss": 0.5111699088199718, - "val_score": -2.6288309752526415 - }, - { - "epoch": 4, - "test_score": -2.5562409044024648, - "train_loss": 0.5082123596389014, - "val_score": -2.5991978251489503 - }, - { - "epoch": 5, - "test_score": -2.5670301819867083, - "train_loss": 0.5054270923137665, - "val_score": -2.6091194256096264 - }, - { - "epoch": 6, - "test_score": -2.5778272977088172, - "train_loss": 0.5041925184898548, - "val_score": -2.6198486463494515 - }, - { - "epoch": 7, - "test_score": -2.5606990160869687, - "train_loss": 0.5048417513434952, - "val_score": -2.602276220164088 - }, - { - "epoch": 8, - "test_score": -2.5618564448879386, - "train_loss": 0.5054062043761348, - "val_score": -2.603846832980074 - }, - { - "epoch": 9, - "test_score": -2.544944827487872, - "train_loss": 0.5027262665129997, - "val_score": -2.58848699643732 - }, - { - "epoch": 10, - "test_score": -2.5682558684288046, - "train_loss": 0.5023538174929919, - "val_score": -2.608500351194885 - }, - { - "epoch": 11, - "test_score": -2.58327123095612, - "train_loss": 0.5022286219639821, - "val_score": -2.624788184929799 - }, - { - "epoch": 12, - "test_score": -2.5471878327692514, - "train_loss": 0.5031755037672885, - "val_score": -2.588843058815793 - }, - { - "epoch": 13, - "test_score": -2.560687470813483, - "train_loss": 0.5017490123843288, - "val_score": -2.602519824330865 - }, - { - "epoch": 14, - "test_score": -2.5609168768116946, - "train_loss": 0.503594289223353, - "val_score": -2.600978201470113 - }, - { - "epoch": 15, - "test_score": -2.570908590376566, - "train_loss": 0.5005721874065228, - "val_score": -2.6146657153116015 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.544944827487872, - "score": -2.544944827487872 - }, - "val": { - "rmse": 2.58848699643732, - "score": -2.58848699643732 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/all_minus_rla-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 62.630138874053955, - "variant": "all_minus_rla" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0.toml deleted file mode 100644 index 8894099eb..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0.toml +++ /dev/null @@ -1,27 +0,0 @@ -dataset = "cooking-time" -variant = "baseline_plr" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0/report.json deleted file mode 100644 index e1a2faf38..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0/report.json +++ /dev/null @@ -1,116 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 3, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "baseline_plr" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.5598227308409616, - "train_loss": 0.562614095640612, - "val_score": -2.600956935110999 - }, - { - "epoch": 1, - "test_score": -2.56268421534399, - "train_loss": 0.49585827218519674, - "val_score": -2.6005418396268225 - }, - { - "epoch": 2, - "test_score": -2.5828534777852807, - "train_loss": 0.4898477091982558, - "val_score": -2.6206401767923544 - }, - { - "epoch": 3, - "test_score": -2.5612615058011223, - "train_loss": 0.48662275988776404, - "val_score": -2.5972627499478844 - }, - { - "epoch": 4, - "test_score": -2.6042154434722344, - "train_loss": 0.48567526619713586, - "val_score": -2.641161350652755 - }, - { - "epoch": 5, - "test_score": -2.5837059880214395, - "train_loss": 0.4844431109256572, - "val_score": -2.617444381633043 - }, - { - "epoch": 6, - "test_score": -2.5854192946362007, - "train_loss": 0.4830809208723876, - "val_score": -2.61894372118841 - }, - { - "epoch": 7, - "test_score": -2.5704376283004158, - "train_loss": 0.4821285430912499, - "val_score": -2.604933287632126 - }, - { - "epoch": 8, - "test_score": -2.597135425642725, - "train_loss": 0.4808750023712983, - "val_score": -2.6334981959278654 - }, - { - "epoch": 9, - "test_score": -2.578672223825131, - "train_loss": 0.4806423933656366, - "val_score": -2.612789181852867 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.5612615058011223, - "score": -2.5612615058011223 - }, - "val": { - "rmse": 2.5972627499478844, - "score": -2.5972627499478844 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 25.95792293548584, - "variant": "baseline_plr" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1.toml deleted file mode 100644 index ca4c2e8a1..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1.toml +++ /dev/null @@ -1,27 +0,0 @@ -dataset = "cooking-time" -variant = "baseline_plr" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1/report.json deleted file mode 100644 index ee4ef6298..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1/report.json +++ /dev/null @@ -1,116 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 3, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "baseline_plr" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.585175370220307, - "train_loss": 0.5666897385507017, - "val_score": -2.6246198651410304 - }, - { - "epoch": 1, - "test_score": -2.5758223914764855, - "train_loss": 0.4957716354915688, - "val_score": -2.6152098514241806 - }, - { - "epoch": 2, - "test_score": -2.5674024068230503, - "train_loss": 0.48955290032936644, - "val_score": -2.6025422688476967 - }, - { - "epoch": 3, - "test_score": -2.5657847681143955, - "train_loss": 0.48720274019885706, - "val_score": -2.601873705696629 - }, - { - "epoch": 4, - "test_score": -2.5715011120963838, - "train_loss": 0.48514884549218257, - "val_score": -2.607710804450608 - }, - { - "epoch": 5, - "test_score": -2.5812347153033297, - "train_loss": 0.48385220712369625, - "val_score": -2.6151459431814095 - }, - { - "epoch": 6, - "test_score": -2.5934737644401955, - "train_loss": 0.4826641380786896, - "val_score": -2.6289346361824175 - }, - { - "epoch": 7, - "test_score": -2.5747132817609746, - "train_loss": 0.48209835602356504, - "val_score": -2.607563966458419 - }, - { - "epoch": 8, - "test_score": -2.574277283809098, - "train_loss": 0.48115245984481264, - "val_score": -2.6069413217336055 - }, - { - "epoch": 9, - "test_score": -2.5774853461928484, - "train_loss": 0.4803670027234533, - "val_score": -2.6114248116625056 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.5657847681143955, - "score": -2.5657847681143955 - }, - "val": { - "rmse": 2.601873705696629, - "score": -2.601873705696629 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 28.450456857681274, - "variant": "baseline_plr" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2.toml deleted file mode 100644 index 21f772c82..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2.toml +++ /dev/null @@ -1,27 +0,0 @@ -dataset = "cooking-time" -variant = "baseline_plr" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2/report.json deleted file mode 100644 index 57677369c..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2/report.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 10, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "baseline_plr" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.5753881550792515, - "train_loss": 0.566168913701633, - "val_score": -2.6158977917760935 - }, - { - "epoch": 1, - "test_score": -2.574276265035827, - "train_loss": 0.4963748224146731, - "val_score": -2.612004217532902 - }, - { - "epoch": 2, - "test_score": -2.5944422521951775, - "train_loss": 0.4892988766098882, - "val_score": -2.630207799527457 - }, - { - "epoch": 3, - "test_score": -2.5944665125966364, - "train_loss": 0.4868296041144981, - "val_score": -2.631199826738509 - }, - { - "epoch": 4, - "test_score": -2.5889661864263944, - "train_loss": 0.48513829654401486, - "val_score": -2.625051770380953 - }, - { - "epoch": 5, - "test_score": -2.5700625034257225, - "train_loss": 0.4835451285581331, - "val_score": -2.6054604228514324 - }, - { - "epoch": 6, - "test_score": -2.5928133456792115, - "train_loss": 0.4826182731100031, - "val_score": -2.6278055729470218 - }, - { - "epoch": 7, - "test_score": -2.5798912702611383, - "train_loss": 0.4817298229213233, - "val_score": -2.6160575591063306 - }, - { - "epoch": 8, - "test_score": -2.5656927734047232, - "train_loss": 0.4806378687824215, - "val_score": -2.6017324947685756 - }, - { - "epoch": 9, - "test_score": -2.5914622869174426, - "train_loss": 0.4803790836720853, - "val_score": -2.62708953162926 - }, - { - "epoch": 10, - "test_score": -2.5666563240272247, - "train_loss": 0.4802004740581856, - "val_score": -2.6010690398654326 - }, - { - "epoch": 11, - "test_score": -2.5836036499682753, - "train_loss": 0.4790540662434724, - "val_score": -2.6167414498627273 - }, - { - "epoch": 12, - "test_score": -2.595542288476678, - "train_loss": 0.4786413835512625, - "val_score": -2.6289079731316325 - }, - { - "epoch": 13, - "test_score": -2.5878016273022295, - "train_loss": 0.47891669821094823, - "val_score": -2.622896441522318 - }, - { - "epoch": 14, - "test_score": -2.585362119711485, - "train_loss": 0.4776747846925581, - "val_score": -2.619510086934299 - }, - { - "epoch": 15, - "test_score": -2.5852033143357676, - "train_loss": 0.47722361324069734, - "val_score": -2.6188749879858246 - }, - { - "epoch": 16, - "test_score": -2.597851371607391, - "train_loss": 0.47689444954330856, - "val_score": -2.6305574896685715 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.5666563240272247, - "score": -2.5666563240272247 - }, - "val": { - "rmse": 2.6010690398654326, - "score": -2.6010690398654326 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/baseline_plr-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 49.608482360839844, - "variant": "baseline_plr" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0.toml deleted file mode 100644 index d16fef85d..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0.toml +++ /dev/null @@ -1,28 +0,0 @@ -dataset = "cooking-time" -variant = "cf_fisd_only" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0/report.json deleted file mode 100644 index b07b4dfc0..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0/report.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "cf_fisd_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.563118185514718, - "train_loss": 0.5551990426338471, - "val_score": -2.6028544565868588 - }, - { - "epoch": 1, - "test_score": -2.572042144127544, - "train_loss": 0.4948291037533734, - "val_score": -2.6100162471981716 - }, - { - "epoch": 2, - "test_score": -2.5773333634568774, - "train_loss": 0.489956555065808, - "val_score": -2.6151421141072886 - }, - { - "epoch": 3, - "test_score": -2.566805130662705, - "train_loss": 0.48730920859285304, - "val_score": -2.6032010437115134 - }, - { - "epoch": 4, - "test_score": -2.610084665613344, - "train_loss": 0.48609176707697344, - "val_score": -2.6468524522057435 - }, - { - "epoch": 5, - "test_score": -2.5852612305505973, - "train_loss": 0.48481034373377896, - "val_score": -2.619681374343505 - }, - { - "epoch": 6, - "test_score": -2.586320554822736, - "train_loss": 0.4835801567580249, - "val_score": -2.620107177988242 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.563118185514718, - "score": -2.563118185514718 - }, - "val": { - "rmse": 2.6028544565868588, - "score": -2.6028544565868588 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 18.744648456573486, - "variant": "cf_fisd_only" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1.toml deleted file mode 100644 index 1e11ee101..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1.toml +++ /dev/null @@ -1,28 +0,0 @@ -dataset = "cooking-time" -variant = "cf_fisd_only" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1/report.json deleted file mode 100644 index 98da7e105..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1/report.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 3, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "cf_fisd_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.5768438681813626, - "train_loss": 0.5615193841156659, - "val_score": -2.6179108032989857 - }, - { - "epoch": 1, - "test_score": -2.5817535758940755, - "train_loss": 0.49551311296385686, - "val_score": -2.6201837952771014 - }, - { - "epoch": 2, - "test_score": -2.5668466500759455, - "train_loss": 0.4899280680729462, - "val_score": -2.6034090287864857 - }, - { - "epoch": 3, - "test_score": -2.5605115851699063, - "train_loss": 0.4877959296510026, - "val_score": -2.5968646915749916 - }, - { - "epoch": 4, - "test_score": -2.572959951018651, - "train_loss": 0.48579797497740734, - "val_score": -2.6096605156120183 - }, - { - "epoch": 5, - "test_score": -2.584487093393811, - "train_loss": 0.48443509812827584, - "val_score": -2.618533662222361 - }, - { - "epoch": 6, - "test_score": -2.598905655830961, - "train_loss": 0.48327650465406813, - "val_score": -2.634865342523215 - }, - { - "epoch": 7, - "test_score": -2.5753543646961994, - "train_loss": 0.48260759662937475, - "val_score": -2.6092616987471433 - }, - { - "epoch": 8, - "test_score": -2.5748974566735727, - "train_loss": 0.48181808666066006, - "val_score": -2.6095718949093794 - }, - { - "epoch": 9, - "test_score": -2.5797974681462987, - "train_loss": 0.48100584772256044, - "val_score": -2.614384850089723 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.5605115851699063, - "score": -2.5605115851699063 - }, - "val": { - "rmse": 2.5968646915749916, - "score": -2.5968646915749916 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 28.90973997116089, - "variant": "cf_fisd_only" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2.toml deleted file mode 100644 index 850658dc6..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2.toml +++ /dev/null @@ -1,28 +0,0 @@ -dataset = "cooking-time" -variant = "cf_fisd_only" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2/report.json deleted file mode 100644 index b164f8c4c..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2/report.json +++ /dev/null @@ -1,159 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 10, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "cf_fisd_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.5875577431837735, - "train_loss": 0.5611629059185853, - "val_score": -2.6295552465826466 - }, - { - "epoch": 1, - "test_score": -2.5747327276980787, - "train_loss": 0.4955460289577106, - "val_score": -2.6127804217806334 - }, - { - "epoch": 2, - "test_score": -2.598434996933457, - "train_loss": 0.489554179681314, - "val_score": -2.6344989389970186 - }, - { - "epoch": 3, - "test_score": -2.593255788760048, - "train_loss": 0.48722675120508346, - "val_score": -2.630046806789206 - }, - { - "epoch": 4, - "test_score": -2.591796323061829, - "train_loss": 0.485682710065498, - "val_score": -2.6285540726041305 - }, - { - "epoch": 5, - "test_score": -2.57461521644028, - "train_loss": 0.4841639764674075, - "val_score": -2.6101172756287023 - }, - { - "epoch": 6, - "test_score": -2.5907818460322876, - "train_loss": 0.4832287656294333, - "val_score": -2.6265027422350546 - }, - { - "epoch": 7, - "test_score": -2.5792667519455312, - "train_loss": 0.4822454852563841, - "val_score": -2.613920079539629 - }, - { - "epoch": 8, - "test_score": -2.567953401107109, - "train_loss": 0.4813956851894791, - "val_score": -2.6037318260814915 - }, - { - "epoch": 9, - "test_score": -2.5921374904230277, - "train_loss": 0.4810924151459256, - "val_score": -2.6279573584396494 - }, - { - "epoch": 10, - "test_score": -2.566079036110905, - "train_loss": 0.48065090662724264, - "val_score": -2.600480321386129 - }, - { - "epoch": 11, - "test_score": -2.5830334725214104, - "train_loss": 0.4798199037710826, - "val_score": -2.6167372586716477 - }, - { - "epoch": 12, - "test_score": -2.594220681743779, - "train_loss": 0.4794894346782753, - "val_score": -2.6278059358636168 - }, - { - "epoch": 13, - "test_score": -2.5928668621350965, - "train_loss": 0.4797732456847354, - "val_score": -2.628311702161112 - }, - { - "epoch": 14, - "test_score": -2.59032379350422, - "train_loss": 0.47852054238319397, - "val_score": -2.6244594380693966 - }, - { - "epoch": 15, - "test_score": -2.585993740242376, - "train_loss": 0.4782114834398837, - "val_score": -2.620114912614322 - }, - { - "epoch": 16, - "test_score": -2.595159492368922, - "train_loss": 0.4778157437706853, - "val_score": -2.629266541322618 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.566079036110905, - "score": -2.566079036110905 - }, - "val": { - "rmse": 2.600480321386129, - "score": -2.600480321386129 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/cf_fisd_only-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 46.023908615112305, - "variant": "cf_fisd_only" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0.toml deleted file mode 100644 index 8afcd8fb2..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "cooking-time" -variant = "esam_only" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0/report.json deleted file mode 100644 index ba2b2b1de..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0/report.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 3, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "esam_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.5576536358788435, - "train_loss": 0.5631172922817437, - "val_score": -2.598798228335147 - }, - { - "epoch": 1, - "test_score": -2.5601839903934147, - "train_loss": 0.49629482143634074, - "val_score": -2.5981515512584874 - }, - { - "epoch": 2, - "test_score": -2.584350375481377, - "train_loss": 0.49001724709261646, - "val_score": -2.6219641069040334 - }, - { - "epoch": 3, - "test_score": -2.5611478448009337, - "train_loss": 0.48687619266209303, - "val_score": -2.5972630253361557 - }, - { - "epoch": 4, - "test_score": -2.6040067903397914, - "train_loss": 0.485992630859753, - "val_score": -2.6409654567124847 - }, - { - "epoch": 5, - "test_score": -2.5828276313573895, - "train_loss": 0.4846961060622791, - "val_score": -2.6166486042962434 - }, - { - "epoch": 6, - "test_score": -2.58425830360799, - "train_loss": 0.4833594560623169, - "val_score": -2.6178357588304086 - }, - { - "epoch": 7, - "test_score": -2.56968583929098, - "train_loss": 0.4823442751222903, - "val_score": -2.6042412607317895 - }, - { - "epoch": 8, - "test_score": -2.5962759380486657, - "train_loss": 0.48112203865437897, - "val_score": -2.6326239543833205 - }, - { - "epoch": 9, - "test_score": -2.577815274216168, - "train_loss": 0.48088436835520976, - "val_score": -2.6120958590596373 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.5611478448009337, - "score": -2.5611478448009337 - }, - "val": { - "rmse": 2.5972630253361557, - "score": -2.5972630253361557 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 36.826345682144165, - "variant": "esam_only" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1.toml deleted file mode 100644 index 33b908233..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "cooking-time" -variant = "esam_only" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1/report.json deleted file mode 100644 index 27b344757..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1/report.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 3, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "esam_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.587351709069442, - "train_loss": 0.5673266920420501, - "val_score": -2.6270270922149246 - }, - { - "epoch": 1, - "test_score": -2.574544003304502, - "train_loss": 0.4962879950935776, - "val_score": -2.614041752342434 - }, - { - "epoch": 2, - "test_score": -2.567338701509846, - "train_loss": 0.48992319713841687, - "val_score": -2.602752321820629 - }, - { - "epoch": 3, - "test_score": -2.563722179453466, - "train_loss": 0.4875023689892915, - "val_score": -2.600100452976782 - }, - { - "epoch": 4, - "test_score": -2.5729477194535098, - "train_loss": 0.4853253735078348, - "val_score": -2.609250368351265 - }, - { - "epoch": 5, - "test_score": -2.5810642018541534, - "train_loss": 0.48425327791823997, - "val_score": -2.6150257805160684 - }, - { - "epoch": 6, - "test_score": -2.594595989463728, - "train_loss": 0.4828814523177104, - "val_score": -2.6301171516753516 - }, - { - "epoch": 7, - "test_score": -2.5756927114027013, - "train_loss": 0.4822785379113378, - "val_score": -2.6085275884408397 - }, - { - "epoch": 8, - "test_score": -2.572590569179888, - "train_loss": 0.4814685883822742, - "val_score": -2.6054338856070185 - }, - { - "epoch": 9, - "test_score": -2.576577479294077, - "train_loss": 0.48072265773206146, - "val_score": -2.6107369711738118 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.563722179453466, - "score": -2.563722179453466 - }, - "val": { - "rmse": 2.600100452976782, - "score": -2.600100452976782 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 39.21153426170349, - "variant": "esam_only" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2.toml deleted file mode 100644 index f6e5c3aa3..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "cooking-time" -variant = "esam_only" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2/report.json deleted file mode 100644 index 42dccb37a..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2/report.json +++ /dev/null @@ -1,160 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 10, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "esam_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.5730321346015366, - "train_loss": 0.566731651087065, - "val_score": -2.6135744577547486 - }, - { - "epoch": 1, - "test_score": -2.5750931917382087, - "train_loss": 0.49662325430560755, - "val_score": -2.61291455716844 - }, - { - "epoch": 2, - "test_score": -2.593852029655047, - "train_loss": 0.48959457659506583, - "val_score": -2.629839840497881 - }, - { - "epoch": 3, - "test_score": -2.59720170482707, - "train_loss": 0.48712443526800686, - "val_score": -2.6338706224915773 - }, - { - "epoch": 4, - "test_score": -2.5891744862235693, - "train_loss": 0.4853177621020927, - "val_score": -2.625148496514752 - }, - { - "epoch": 5, - "test_score": -2.5708206741128494, - "train_loss": 0.48389967470555695, - "val_score": -2.606276265476257 - }, - { - "epoch": 6, - "test_score": -2.5933733747115633, - "train_loss": 0.4827877854978716, - "val_score": -2.6281835234268742 - }, - { - "epoch": 7, - "test_score": -2.5794014285161495, - "train_loss": 0.4820292495392464, - "val_score": -2.615673754749277 - }, - { - "epoch": 8, - "test_score": -2.5661116478848696, - "train_loss": 0.48090581152890177, - "val_score": -2.60218120927395 - }, - { - "epoch": 9, - "test_score": -2.5917589749725156, - "train_loss": 0.48059917328593965, - "val_score": -2.627395172931714 - }, - { - "epoch": 10, - "test_score": -2.56671335830164, - "train_loss": 0.4804097650287388, - "val_score": -2.601435477790095 - }, - { - "epoch": 11, - "test_score": -2.584341611265953, - "train_loss": 0.4794119491770461, - "val_score": -2.6174121361735105 - }, - { - "epoch": 12, - "test_score": -2.595533745766955, - "train_loss": 0.4789287713733879, - "val_score": -2.6290037411981997 - }, - { - "epoch": 13, - "test_score": -2.587051842988804, - "train_loss": 0.47913944774919803, - "val_score": -2.622268709032895 - }, - { - "epoch": 14, - "test_score": -2.585114869714288, - "train_loss": 0.47785790546520335, - "val_score": -2.6195328409546783 - }, - { - "epoch": 15, - "test_score": -2.5854203090186108, - "train_loss": 0.47741706763301883, - "val_score": -2.619167751595635 - }, - { - "epoch": 16, - "test_score": -2.598019039662804, - "train_loss": 0.47704946135615445, - "val_score": -2.630867440509398 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.56671335830164, - "score": -2.56671335830164 - }, - "val": { - "rmse": 2.601435477790095, - "score": -2.601435477790095 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/esam_only-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 66.56187295913696, - "variant": "esam_only" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0.toml deleted file mode 100644 index ffa62de37..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "cooking-time" -variant = "mfb_only" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0/report.json deleted file mode 100644 index f8cefbf30..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0/report.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 5, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "mfb_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.5661236333044846, - "train_loss": 0.5758807992076015, - "val_score": -2.6119257172541515 - }, - { - "epoch": 1, - "test_score": -2.5743387799216704, - "train_loss": 0.5163963157851417, - "val_score": -2.618051870148412 - }, - { - "epoch": 2, - "test_score": -2.5601411522753925, - "train_loss": 0.5085933527967952, - "val_score": -2.6044778154775474 - }, - { - "epoch": 3, - "test_score": -2.576163730796452, - "train_loss": 0.5093893889371339, - "val_score": -2.619102301377666 - }, - { - "epoch": 4, - "test_score": -2.57161255397797, - "train_loss": 0.505496577905105, - "val_score": -2.6141280326221903 - }, - { - "epoch": 5, - "test_score": -2.5555160998555837, - "train_loss": 0.5060446619450509, - "val_score": -2.5961102696644223 - }, - { - "epoch": 6, - "test_score": -2.5676436554743733, - "train_loss": 0.5035226178598834, - "val_score": -2.6076490896143696 - }, - { - "epoch": 7, - "test_score": -2.5625663376571928, - "train_loss": 0.5044513223944483, - "val_score": -2.6047008017490376 - }, - { - "epoch": 8, - "test_score": -2.583804446506284, - "train_loss": 0.50434070288598, - "val_score": -2.6261775690664138 - }, - { - "epoch": 9, - "test_score": -2.585765452582473, - "train_loss": 0.5042245648465715, - "val_score": -2.627743604203495 - }, - { - "epoch": 10, - "test_score": -2.5881280293524083, - "train_loss": 0.5026848740405865, - "val_score": -2.6263449720071628 - }, - { - "epoch": 11, - "test_score": -2.567627870074032, - "train_loss": 0.501694805718757, - "val_score": -2.6076158087112957 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.5555160998555837, - "score": -2.5555160998555837 - }, - "val": { - "rmse": 2.5961102696644223, - "score": -2.5961102696644223 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 30.617911338806152, - "variant": "mfb_only" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1.toml deleted file mode 100644 index c953cfa7f..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "cooking-time" -variant = "mfb_only" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1/report.json deleted file mode 100644 index 3ba3c8265..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1/report.json +++ /dev/null @@ -1,142 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 7, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "mfb_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.5797754726740076, - "train_loss": 0.5817791330921758, - "val_score": -2.623108227566879 - }, - { - "epoch": 1, - "test_score": -2.5741110331989514, - "train_loss": 0.518071799664884, - "val_score": -2.618382423113911 - }, - { - "epoch": 2, - "test_score": -2.572947534126318, - "train_loss": 0.5103391008871095, - "val_score": -2.6161554379786542 - }, - { - "epoch": 3, - "test_score": -2.558461426086861, - "train_loss": 0.506716252715738, - "val_score": -2.6023681126430156 - }, - { - "epoch": 4, - "test_score": -2.5548843167984927, - "train_loss": 0.5062507541330011, - "val_score": -2.5978860624377282 - }, - { - "epoch": 5, - "test_score": -2.5596182832148666, - "train_loss": 0.5055183303785754, - "val_score": -2.6004166929414816 - }, - { - "epoch": 6, - "test_score": -2.561558620314518, - "train_loss": 0.5038739445510211, - "val_score": -2.603257826842896 - }, - { - "epoch": 7, - "test_score": -2.539276959092014, - "train_loss": 0.5045690974136731, - "val_score": -2.578839659618401 - }, - { - "epoch": 8, - "test_score": -2.55441917386838, - "train_loss": 0.5041662433125951, - "val_score": -2.5950167216973132 - }, - { - "epoch": 9, - "test_score": -2.5487520754638724, - "train_loss": 0.5028662115066975, - "val_score": -2.5891763278800166 - }, - { - "epoch": 10, - "test_score": -2.5801767219226126, - "train_loss": 0.5012965699036916, - "val_score": -2.6218624437506626 - }, - { - "epoch": 11, - "test_score": -2.5697164569101383, - "train_loss": 0.5027571734007414, - "val_score": -2.610265340069635 - }, - { - "epoch": 12, - "test_score": -2.5612925964655955, - "train_loss": 0.5003397878225859, - "val_score": -2.602886882391948 - }, - { - "epoch": 13, - "test_score": -2.598473900627761, - "train_loss": 0.5009741587144835, - "val_score": -2.6391601110805962 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.539276959092014, - "score": -2.539276959092014 - }, - "val": { - "rmse": 2.578839659618401, - "score": -2.578839659618401 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 39.26718091964722, - "variant": "mfb_only" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2.toml deleted file mode 100644 index fac1025ac..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "cooking-time" -variant = "mfb_only" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2/report.json deleted file mode 100644 index d5647b82d..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2/report.json +++ /dev/null @@ -1,142 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 7, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "mfb_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.5842396674188115, - "train_loss": 0.5787746052484255, - "val_score": -2.6294725553506053 - }, - { - "epoch": 1, - "test_score": -2.5609527196426085, - "train_loss": 0.5174002875615885, - "val_score": -2.6062741614669447 - }, - { - "epoch": 2, - "test_score": -2.5834044067286164, - "train_loss": 0.511275235060099, - "val_score": -2.6247500346015094 - }, - { - "epoch": 3, - "test_score": -2.576294682766308, - "train_loss": 0.5079844250335349, - "val_score": -2.6197295184692186 - }, - { - "epoch": 4, - "test_score": -2.573368285127898, - "train_loss": 0.5065022978159759, - "val_score": -2.614125205300859 - }, - { - "epoch": 5, - "test_score": -2.5507163817962533, - "train_loss": 0.5062107411590783, - "val_score": -2.5932705907251505 - }, - { - "epoch": 6, - "test_score": -2.5672484341318604, - "train_loss": 0.5054079772115828, - "val_score": -2.606842731064844 - }, - { - "epoch": 7, - "test_score": -2.5386993618023146, - "train_loss": 0.505251935204944, - "val_score": -2.577726668594756 - }, - { - "epoch": 8, - "test_score": -2.5609610984138573, - "train_loss": 0.5025441719068063, - "val_score": -2.6023546450549193 - }, - { - "epoch": 9, - "test_score": -2.562959676311487, - "train_loss": 0.5021548641694559, - "val_score": -2.6056102159519456 - }, - { - "epoch": 10, - "test_score": -2.5657840247360215, - "train_loss": 0.5002038159348943, - "val_score": -2.607008357600551 - }, - { - "epoch": 11, - "test_score": -2.574676334075847, - "train_loss": 0.4997164035165632, - "val_score": -2.6152728464660053 - }, - { - "epoch": 12, - "test_score": -2.5638825943321844, - "train_loss": 0.5002949726474177, - "val_score": -2.6052825266388324 - }, - { - "epoch": 13, - "test_score": -2.5865687035005402, - "train_loss": 0.49844017893344433, - "val_score": -2.6283478051631612 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.5386993618023146, - "score": -2.5386993618023146 - }, - "val": { - "rmse": 2.577726668594756, - "score": -2.577726668594756 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/mfb_only-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 41.57825565338135, - "variant": "mfb_only" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0.toml deleted file mode 100644 index 689ec16cd..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0.toml +++ /dev/null @@ -1,30 +0,0 @@ -dataset = "cooking-time" -variant = "rla_only" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0/report.json deleted file mode 100644 index fbf3b0516..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0/report.json +++ /dev/null @@ -1,119 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 3, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "rla_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.6067414842618137, - "train_loss": 0.5621826989693685, - "val_score": -2.648587653030598 - }, - { - "epoch": 1, - "test_score": -2.589142164939684, - "train_loss": 0.49587324476456857, - "val_score": -2.626401890207494 - }, - { - "epoch": 2, - "test_score": -2.5718627706911366, - "train_loss": 0.48924584055806064, - "val_score": -2.606097327135619 - }, - { - "epoch": 3, - "test_score": -2.5699112877582024, - "train_loss": 0.4867811350672095, - "val_score": -2.6052357628095346 - }, - { - "epoch": 4, - "test_score": -2.5745107574860464, - "train_loss": 0.485320704208838, - "val_score": -2.609898040796941 - }, - { - "epoch": 5, - "test_score": -2.587636798430576, - "train_loss": 0.4839465403342032, - "val_score": -2.624125743783909 - }, - { - "epoch": 6, - "test_score": -2.584093986833313, - "train_loss": 0.48281669187116194, - "val_score": -2.619210170473965 - }, - { - "epoch": 7, - "test_score": -2.5796617956053485, - "train_loss": 0.4819454647399284, - "val_score": -2.612544710059628 - }, - { - "epoch": 8, - "test_score": -2.5948246947005935, - "train_loss": 0.4810482220606761, - "val_score": -2.630481083894551 - }, - { - "epoch": 9, - "test_score": -2.590343490417043, - "train_loss": 0.4809136336988157, - "val_score": -2.6232398353437167 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.5699112877582024, - "score": -2.5699112877582024 - }, - "val": { - "rmse": 2.6052357628095346, - "score": -2.6052357628095346 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 28.367936372756958, - "variant": "rla_only" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1.toml deleted file mode 100644 index 0f92ae19e..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1.toml +++ /dev/null @@ -1,30 +0,0 @@ -dataset = "cooking-time" -variant = "rla_only" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1/report.json deleted file mode 100644 index f3d4ce614..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1/report.json +++ /dev/null @@ -1,119 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 3, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "rla_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.574028969051126, - "train_loss": 0.5646411073100459, - "val_score": -2.6166735699637482 - }, - { - "epoch": 1, - "test_score": -2.5752987253186155, - "train_loss": 0.49558495642902617, - "val_score": -2.6112346304317455 - }, - { - "epoch": 2, - "test_score": -2.553700761063615, - "train_loss": 0.48945968484019375, - "val_score": -2.592604510592764 - }, - { - "epoch": 3, - "test_score": -2.5449543831564863, - "train_loss": 0.4868325668949265, - "val_score": -2.5806605037686903 - }, - { - "epoch": 4, - "test_score": -2.5839795770694307, - "train_loss": 0.48536315828830273, - "val_score": -2.6187721124047783 - }, - { - "epoch": 5, - "test_score": -2.5676175630837914, - "train_loss": 0.4838317922643713, - "val_score": -2.6051026048423873 - }, - { - "epoch": 6, - "test_score": -2.583028119012807, - "train_loss": 0.4833074563795382, - "val_score": -2.6181113363361583 - }, - { - "epoch": 7, - "test_score": -2.575665219499035, - "train_loss": 0.48161201272998844, - "val_score": -2.6116000071743652 - }, - { - "epoch": 8, - "test_score": -2.5636876773187365, - "train_loss": 0.4816002647082011, - "val_score": -2.597791074538032 - }, - { - "epoch": 9, - "test_score": -2.5757077068628393, - "train_loss": 0.4801698541856027, - "val_score": -2.612110097892733 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.5449543831564863, - "score": -2.5449543831564863 - }, - "val": { - "rmse": 2.5806605037686903, - "score": -2.5806605037686903 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 30.289871215820312, - "variant": "rla_only" -} diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2.toml deleted file mode 100644 index 1187840b4..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2.toml +++ /dev/null @@ -1,30 +0,0 @@ -dataset = "cooking-time" -variant = "rla_only" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/cooking-time" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2/report.json deleted file mode 100644 index e9954ed1f..000000000 --- a/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2/report.json +++ /dev/null @@ -1,107 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 1, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/cooking-time" - }, - "dataset": "cooking-time", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "rla_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2.toml", - "dataset": "cooking-time", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -2.6212606590101575, - "train_loss": 0.5664722629495569, - "val_score": -2.6634027553513446 - }, - { - "epoch": 1, - "test_score": -2.5677737420490234, - "train_loss": 0.49630044951095237, - "val_score": -2.6059867195323827 - }, - { - "epoch": 2, - "test_score": -2.5929284690585463, - "train_loss": 0.48952298738934974, - "val_score": -2.6290505356892178 - }, - { - "epoch": 3, - "test_score": -2.5822994768076653, - "train_loss": 0.48763485615317886, - "val_score": -2.6185297470515487 - }, - { - "epoch": 4, - "test_score": -2.5928140813080858, - "train_loss": 0.4848814955702773, - "val_score": -2.628734384618246 - }, - { - "epoch": 5, - "test_score": -2.5781435879123955, - "train_loss": 0.48458673368703137, - "val_score": -2.6120996013245725 - }, - { - "epoch": 6, - "test_score": -2.582673655134024, - "train_loss": 0.48272706715910285, - "val_score": -2.6179256480057504 - }, - { - "epoch": 7, - "test_score": -2.579921674352669, - "train_loss": 0.4819437984947686, - "val_score": -2.6134262158794432 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 2.5677737420490234, - "score": -2.5677737420490234 - }, - "val": { - "rmse": 2.6059867195323827, - "score": -2.6059867195323827 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/cooking-time/rla_only-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 25.377673625946045, - "variant": "rla_only" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0.toml deleted file mode 100644 index 2ea3ddbd7..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0.toml +++ /dev/null @@ -1,35 +0,0 @@ -dataset = "delivery-eta" -variant = "all_four_combined" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0/report.json deleted file mode 100644 index 70ac8fe4f..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0/report.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_four_combined" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.8656861082768745, - "train_loss": 0.6351582882178091, - "val_score": -1.7428132032666934 - }, - { - "epoch": 1, - "test_score": -1.9173332451200045, - "train_loss": 0.5862035707835733, - "val_score": -1.8012485518439756 - }, - { - "epoch": 2, - "test_score": -1.9211029735094005, - "train_loss": 0.5774363997208811, - "val_score": -1.8070825974375915 - }, - { - "epoch": 3, - "test_score": -1.9158700932442214, - "train_loss": 0.5784730580601379, - "val_score": -1.806458566101268 - }, - { - "epoch": 4, - "test_score": -1.9295777818631115, - "train_loss": 0.5680552694919335, - "val_score": -1.8254905694547126 - }, - { - "epoch": 5, - "test_score": -1.900368983428771, - "train_loss": 0.5624197795443291, - "val_score": -1.8009521006135125 - }, - { - "epoch": 6, - "test_score": -1.9369851935849267, - "train_loss": 0.55270509258674, - "val_score": -1.8281875338822213 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.8656861082768745, - "score": -1.8656861082768745 - }, - "val": { - "rmse": 1.7428132032666934, - "score": -1.7428132032666934 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 37.28859186172485, - "variant": "all_four_combined" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1.toml deleted file mode 100644 index 5d9c87b30..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1.toml +++ /dev/null @@ -1,35 +0,0 @@ -dataset = "delivery-eta" -variant = "all_four_combined" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1/report.json deleted file mode 100644 index 42a14dd3f..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1/report.json +++ /dev/null @@ -1,112 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 1, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_four_combined" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.9131975360467681, - "train_loss": 0.6403110505890672, - "val_score": -1.8000882153617102 - }, - { - "epoch": 1, - "test_score": -1.8804933819744376, - "train_loss": 0.595119994922276, - "val_score": -1.770191195878174 - }, - { - "epoch": 2, - "test_score": -1.908992732814027, - "train_loss": 0.5801966007608567, - "val_score": -1.7980903164432638 - }, - { - "epoch": 3, - "test_score": -1.922019580892163, - "train_loss": 0.5665582211783332, - "val_score": -1.8065416463596233 - }, - { - "epoch": 4, - "test_score": -1.9283695430647496, - "train_loss": 0.5643883639008459, - "val_score": -1.8264561396192909 - }, - { - "epoch": 5, - "test_score": -1.8923228813143904, - "train_loss": 0.564144954629188, - "val_score": -1.7846969581263876 - }, - { - "epoch": 6, - "test_score": -1.9317067195743272, - "train_loss": 0.5586930552538294, - "val_score": -1.8305701181735838 - }, - { - "epoch": 7, - "test_score": -1.9308092200702978, - "train_loss": 0.5511535132018319, - "val_score": -1.8270347800449493 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.8804933819744376, - "score": -1.8804933819744376 - }, - "val": { - "rmse": 1.770191195878174, - "score": -1.770191195878174 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 43.716530084609985, - "variant": "all_four_combined" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2.toml deleted file mode 100644 index df6430d22..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2.toml +++ /dev/null @@ -1,35 +0,0 @@ -dataset = "delivery-eta" -variant = "all_four_combined" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2/report.json deleted file mode 100644 index ab7cd1f8b..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2/report.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_four_combined" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.8965344093393364, - "train_loss": 0.6273651157852507, - "val_score": -1.776750738124176 - }, - { - "epoch": 1, - "test_score": -1.9010519854818586, - "train_loss": 0.5775207037473247, - "val_score": -1.7887463019524075 - }, - { - "epoch": 2, - "test_score": -1.9282420687013453, - "train_loss": 0.578986707830081, - "val_score": -1.8215971903394392 - }, - { - "epoch": 3, - "test_score": -1.9296743415661881, - "train_loss": 0.5668568354453484, - "val_score": -1.8227773849319975 - }, - { - "epoch": 4, - "test_score": -1.9421842002687175, - "train_loss": 0.5612639053894656, - "val_score": -1.8275582502816723 - }, - { - "epoch": 5, - "test_score": -1.8922988795820175, - "train_loss": 0.564061529444952, - "val_score": -1.7890262516797284 - }, - { - "epoch": 6, - "test_score": -1.886318187959984, - "train_loss": 0.5608200274244712, - "val_score": -1.782879134595343 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.8965344093393364, - "score": -1.8965344093393364 - }, - "val": { - "rmse": 1.776750738124176, - "score": -1.776750738124176 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_four_combined-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 38.261871099472046, - "variant": "all_four_combined" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0.toml deleted file mode 100644 index 1620a13b0..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0.toml +++ /dev/null @@ -1,34 +0,0 @@ -dataset = "delivery-eta" -variant = "all_minus_cf_fisd" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0/report.json deleted file mode 100644 index 04525af19..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0/report.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_minus_cf_fisd" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.891667480580607, - "train_loss": 0.6645825640128477, - "val_score": -1.7783494723849997 - }, - { - "epoch": 1, - "test_score": -1.9410785728560749, - "train_loss": 0.5880636803425141, - "val_score": -1.8319506271614023 - }, - { - "epoch": 2, - "test_score": -1.8896939215496575, - "train_loss": 0.5807777503981207, - "val_score": -1.7790821335280576 - }, - { - "epoch": 3, - "test_score": -1.9174438504150855, - "train_loss": 0.585850615571015, - "val_score": -1.800542852877492 - }, - { - "epoch": 4, - "test_score": -1.885224566148601, - "train_loss": 0.5778306814005775, - "val_score": -1.7802418315214172 - }, - { - "epoch": 5, - "test_score": -1.9026999012896584, - "train_loss": 0.5774297648972838, - "val_score": -1.8049066583903193 - }, - { - "epoch": 6, - "test_score": -1.9138769565881466, - "train_loss": 0.5662677301107532, - "val_score": -1.8001846351236526 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.891667480580607, - "score": -1.891667480580607 - }, - "val": { - "rmse": 1.7783494723849997, - "score": -1.7783494723849997 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 38.053216218948364, - "variant": "all_minus_cf_fisd" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1.toml deleted file mode 100644 index a96c2c016..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1.toml +++ /dev/null @@ -1,34 +0,0 @@ -dataset = "delivery-eta" -variant = "all_minus_cf_fisd" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1/report.json deleted file mode 100644 index ceb35433c..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1/report.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 1, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_minus_cf_fisd" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.9505100292387032, - "train_loss": 0.6705632031398968, - "val_score": -1.8468169872899822 - }, - { - "epoch": 1, - "test_score": -1.8636960381640717, - "train_loss": 0.5966045582381478, - "val_score": -1.759412249480788 - }, - { - "epoch": 2, - "test_score": -1.906402769377118, - "train_loss": 0.5835885000924994, - "val_score": -1.7950902949072276 - }, - { - "epoch": 3, - "test_score": -1.8905796644591906, - "train_loss": 0.5713840862260248, - "val_score": -1.7731201291210676 - }, - { - "epoch": 4, - "test_score": -1.9279880221536947, - "train_loss": 0.5733988989878746, - "val_score": -1.8203217338360556 - }, - { - "epoch": 5, - "test_score": -1.9064170889244985, - "train_loss": 0.578271459271438, - "val_score": -1.7922601899602175 - }, - { - "epoch": 6, - "test_score": -1.922968979520343, - "train_loss": 0.5738831989086457, - "val_score": -1.8224447965699262 - }, - { - "epoch": 7, - "test_score": -1.9305441495465439, - "train_loss": 0.5651097101886777, - "val_score": -1.8236467334566928 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.8636960381640717, - "score": -1.8636960381640717 - }, - "val": { - "rmse": 1.759412249480788, - "score": -1.759412249480788 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 43.60566973686218, - "variant": "all_minus_cf_fisd" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2.toml deleted file mode 100644 index 43d746891..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2.toml +++ /dev/null @@ -1,34 +0,0 @@ -dataset = "delivery-eta" -variant = "all_minus_cf_fisd" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2/report.json deleted file mode 100644 index 6a9d80a6b..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2/report.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 1, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_minus_cf_fisd" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.8714515168791568, - "train_loss": 0.6583039582210736, - "val_score": -1.757904253790227 - }, - { - "epoch": 1, - "test_score": -1.8633819488889196, - "train_loss": 0.578686815108696, - "val_score": -1.7545374221006966 - }, - { - "epoch": 2, - "test_score": -1.9262881588730187, - "train_loss": 0.5821841881222969, - "val_score": -1.8210502079259065 - }, - { - "epoch": 3, - "test_score": -1.938032384761293, - "train_loss": 0.5721039728526651, - "val_score": -1.826789824842461 - }, - { - "epoch": 4, - "test_score": -1.9325213916971438, - "train_loss": 0.5703568275827561, - "val_score": -1.8151809333173128 - }, - { - "epoch": 5, - "test_score": -1.9152714862063926, - "train_loss": 0.5760056446068478, - "val_score": -1.8096439616269226 - }, - { - "epoch": 6, - "test_score": -1.890280637038989, - "train_loss": 0.5767811636855132, - "val_score": -1.7839441504139821 - }, - { - "epoch": 7, - "test_score": -1.9192103079194238, - "train_loss": 0.5665370217204964, - "val_score": -1.8114721080160514 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.8633819488889196, - "score": -1.8633819488889196 - }, - "val": { - "rmse": 1.7545374221006966, - "score": -1.7545374221006966 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_cf_fisd-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 43.14394950866699, - "variant": "all_minus_cf_fisd" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0.toml deleted file mode 100644 index 72936a3e7..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "delivery-eta" -variant = "all_minus_esam" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0/report.json deleted file mode 100644 index 6569eeaab..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0/report.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_minus_esam" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.8574282929864145, - "train_loss": 0.6346418135357599, - "val_score": -1.734733819358604 - }, - { - "epoch": 1, - "test_score": -1.9188538663007366, - "train_loss": 0.5895762147694609, - "val_score": -1.8017549674986006 - }, - { - "epoch": 2, - "test_score": -1.921810118037487, - "train_loss": 0.5745818023263973, - "val_score": -1.8123889099803876 - }, - { - "epoch": 3, - "test_score": -1.8605371257519345, - "train_loss": 0.576130695586657, - "val_score": -1.7577453600459139 - }, - { - "epoch": 4, - "test_score": -1.944784069676435, - "train_loss": 0.5613377011605423, - "val_score": -1.8333236592933424 - }, - { - "epoch": 5, - "test_score": -1.9003918168474871, - "train_loss": 0.5654894726989913, - "val_score": -1.7906525055535318 - }, - { - "epoch": 6, - "test_score": -1.9127423151629934, - "train_loss": 0.5558785330640138, - "val_score": -1.8136087512193444 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.8574282929864145, - "score": -1.8574282929864145 - }, - "val": { - "rmse": 1.734733819358604, - "score": -1.734733819358604 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 25.650846242904663, - "variant": "all_minus_esam" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1.toml deleted file mode 100644 index 475232b10..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "delivery-eta" -variant = "all_minus_esam" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1/report.json deleted file mode 100644 index 26eb815f7..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1/report.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 3, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_minus_esam" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.9402282948223006, - "train_loss": 0.6321390519176957, - "val_score": -1.8233988387610958 - }, - { - "epoch": 1, - "test_score": -1.913040261845467, - "train_loss": 0.5818619223406715, - "val_score": -1.8055431104703896 - }, - { - "epoch": 2, - "test_score": -1.8981380775963306, - "train_loss": 0.5805560267754715, - "val_score": -1.78609603752208 - }, - { - "epoch": 3, - "test_score": -1.8896128569809156, - "train_loss": 0.5727628110534083, - "val_score": -1.7733117953883153 - }, - { - "epoch": 4, - "test_score": -1.914617652571641, - "train_loss": 0.561080602398754, - "val_score": -1.8137358036971076 - }, - { - "epoch": 5, - "test_score": -1.917097900087658, - "train_loss": 0.5638521348472929, - "val_score": -1.8097799213716703 - }, - { - "epoch": 6, - "test_score": -1.899768378545227, - "train_loss": 0.5505518221507107, - "val_score": -1.7935067493704147 - }, - { - "epoch": 7, - "test_score": -1.9753858961878896, - "train_loss": 0.5567100800736977, - "val_score": -1.8744500943284486 - }, - { - "epoch": 8, - "test_score": -1.9289701417992142, - "train_loss": 0.5494376259998683, - "val_score": -1.8191711383903835 - }, - { - "epoch": 9, - "test_score": -1.9488545232297387, - "train_loss": 0.5475168900333182, - "val_score": -1.8421066201714127 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.8896128569809156, - "score": -1.8896128569809156 - }, - "val": { - "rmse": 1.7733117953883153, - "score": -1.7733117953883153 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 37.29048800468445, - "variant": "all_minus_esam" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2.toml deleted file mode 100644 index eb4d035c6..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "delivery-eta" -variant = "all_minus_esam" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2/report.json deleted file mode 100644 index 80251e8da..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2/report.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_minus_esam" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.889693353794343, - "train_loss": 0.6369702032882801, - "val_score": -1.7678478056315658 - }, - { - "epoch": 1, - "test_score": -1.928198915916465, - "train_loss": 0.5902615276566388, - "val_score": -1.8122465022165561 - }, - { - "epoch": 2, - "test_score": -1.9130482380274412, - "train_loss": 0.5736563127406322, - "val_score": -1.7966875724897298 - }, - { - "epoch": 3, - "test_score": -1.9379275680800756, - "train_loss": 0.5651734101511267, - "val_score": -1.8271425655888411 - }, - { - "epoch": 4, - "test_score": -1.9250888779174509, - "train_loss": 0.570913371378488, - "val_score": -1.8123990392518639 - }, - { - "epoch": 5, - "test_score": -1.913662366498882, - "train_loss": 0.5671725908335108, - "val_score": -1.8027153303436068 - }, - { - "epoch": 6, - "test_score": -1.8924741293223586, - "train_loss": 0.5528815042798536, - "val_score": -1.7809590553790522 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.889693353794343, - "score": -1.889693353794343 - }, - "val": { - "rmse": 1.7678478056315658, - "score": -1.7678478056315658 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_esam-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 26.155076503753662, - "variant": "all_minus_esam" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0.toml deleted file mode 100644 index 0b7668306..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "delivery-eta" -variant = "all_minus_mfb" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0/report.json deleted file mode 100644 index 8b15b2ac3..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0/report.json +++ /dev/null @@ -1,128 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 4, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_minus_mfb" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.888090720801521, - "train_loss": 0.5960141698809436, - "val_score": -1.7678867133944087 - }, - { - "epoch": 1, - "test_score": -1.9079790872111537, - "train_loss": 0.545285747434101, - "val_score": -1.8052117713268874 - }, - { - "epoch": 2, - "test_score": -1.8922816183054478, - "train_loss": 0.5397576872449722, - "val_score": -1.7867413251257125 - }, - { - "epoch": 3, - "test_score": -1.9028843419042973, - "train_loss": 0.536361330399548, - "val_score": -1.7928649606355145 - }, - { - "epoch": 4, - "test_score": -1.8611355949170485, - "train_loss": 0.5343982783982354, - "val_score": -1.7538454856626753 - }, - { - "epoch": 5, - "test_score": -1.8868397423153196, - "train_loss": 0.5325560691582896, - "val_score": -1.7828675671965248 - }, - { - "epoch": 6, - "test_score": -1.9080221349963706, - "train_loss": 0.5307190265968769, - "val_score": -1.8020101396400294 - }, - { - "epoch": 7, - "test_score": -1.9162634826305283, - "train_loss": 0.5294287024188216, - "val_score": -1.8024161441254358 - }, - { - "epoch": 8, - "test_score": -1.9172990488246484, - "train_loss": 0.5285489454756688, - "val_score": -1.8042130940772272 - }, - { - "epoch": 9, - "test_score": -1.8960975700374516, - "train_loss": 0.5278546320695947, - "val_score": -1.792183697936757 - }, - { - "epoch": 10, - "test_score": -1.8802992639365812, - "train_loss": 0.5262137990798393, - "val_score": -1.7697711975578432 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.8611355949170485, - "score": -1.8611355949170485 - }, - "val": { - "rmse": 1.7538454856626753, - "score": -1.7538454856626753 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 58.886919260025024, - "variant": "all_minus_mfb" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1.toml deleted file mode 100644 index bb9ee9d12..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "delivery-eta" -variant = "all_minus_mfb" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1/report.json deleted file mode 100644 index fdd5787b4..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1/report.json +++ /dev/null @@ -1,116 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 2, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_minus_mfb" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.9291432960802894, - "train_loss": 0.5934300527085353, - "val_score": -1.8205802622313019 - }, - { - "epoch": 1, - "test_score": -1.8930283710632847, - "train_loss": 0.544890384604461, - "val_score": -1.7886505318955528 - }, - { - "epoch": 2, - "test_score": -1.8741570167185646, - "train_loss": 0.5395748837150797, - "val_score": -1.7677035629696638 - }, - { - "epoch": 3, - "test_score": -1.9244894837002682, - "train_loss": 0.5360066024926458, - "val_score": -1.815649518948847 - }, - { - "epoch": 4, - "test_score": -1.9197741555463843, - "train_loss": 0.5336031124104549, - "val_score": -1.8135653686143167 - }, - { - "epoch": 5, - "test_score": -1.927351367627674, - "train_loss": 0.5316786544166342, - "val_score": -1.8143061484933616 - }, - { - "epoch": 6, - "test_score": -1.919555504924725, - "train_loss": 0.5302330420835175, - "val_score": -1.8160835225522036 - }, - { - "epoch": 7, - "test_score": -1.9236640356734735, - "train_loss": 0.5287505892941552, - "val_score": -1.8162988774548237 - }, - { - "epoch": 8, - "test_score": -1.945460608792154, - "train_loss": 0.5278126967214319, - "val_score": -1.8351122360830159 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.8741570167185646, - "score": -1.8741570167185646 - }, - "val": { - "rmse": 1.7677035629696638, - "score": -1.7677035629696638 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 47.90096688270569, - "variant": "all_minus_mfb" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2.toml deleted file mode 100644 index a02615d6f..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "delivery-eta" -variant = "all_minus_mfb" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2/report.json deleted file mode 100644 index 428426e0a..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2/report.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_minus_mfb" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.870944405411248, - "train_loss": 0.5949648753569944, - "val_score": -1.7558893831296303 - }, - { - "epoch": 1, - "test_score": -1.898728148263703, - "train_loss": 0.5453242161413179, - "val_score": -1.7877388934576073 - }, - { - "epoch": 2, - "test_score": -1.9169362818397024, - "train_loss": 0.540534051230354, - "val_score": -1.8111667333277426 - }, - { - "epoch": 3, - "test_score": -1.9006590854507406, - "train_loss": 0.5365165554694016, - "val_score": -1.7956854822081687 - }, - { - "epoch": 4, - "test_score": -1.9391828427407138, - "train_loss": 0.5344064231336552, - "val_score": -1.8293733165799688 - }, - { - "epoch": 5, - "test_score": -1.9016114378299895, - "train_loss": 0.532535148580579, - "val_score": -1.793748806411399 - }, - { - "epoch": 6, - "test_score": -1.9103862101360687, - "train_loss": 0.5309882124845129, - "val_score": -1.7998801270941822 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.870944405411248, - "score": -1.870944405411248 - }, - "val": { - "rmse": 1.7558893831296303, - "score": -1.7558893831296303 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_mfb-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 38.0086407661438, - "variant": "all_minus_mfb" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0.toml deleted file mode 100644 index a8e54a528..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0.toml +++ /dev/null @@ -1,32 +0,0 @@ -dataset = "delivery-eta" -variant = "all_minus_rla" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0/report.json deleted file mode 100644 index 438690292..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0/report.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_minus_rla" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.860688971411864, - "train_loss": 0.6334640344564062, - "val_score": -1.7427255800271264 - }, - { - "epoch": 1, - "test_score": -1.9110673163198892, - "train_loss": 0.587792286472599, - "val_score": -1.7940440558272999 - }, - { - "epoch": 2, - "test_score": -1.905961249322842, - "train_loss": 0.5795968405521699, - "val_score": -1.7906092325152145 - }, - { - "epoch": 3, - "test_score": -1.9461060403001613, - "train_loss": 0.5841769684840293, - "val_score": -1.8283881625656664 - }, - { - "epoch": 4, - "test_score": -1.9359065240409257, - "train_loss": 0.5748188164982483, - "val_score": -1.8243624448850637 - }, - { - "epoch": 5, - "test_score": -1.9230347522366662, - "train_loss": 0.5720745399050469, - "val_score": -1.8228242759906252 - }, - { - "epoch": 6, - "test_score": -1.9510359922028844, - "train_loss": 0.5629213082529333, - "val_score": -1.8318482655318722 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.860688971411864, - "score": -1.860688971411864 - }, - "val": { - "rmse": 1.7427255800271264, - "score": -1.7427255800271264 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 33.61688852310181, - "variant": "all_minus_rla" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1.toml deleted file mode 100644 index b36c0e027..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1.toml +++ /dev/null @@ -1,32 +0,0 @@ -dataset = "delivery-eta" -variant = "all_minus_rla" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1/report.json deleted file mode 100644 index 9f58cc0b6..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1/report.json +++ /dev/null @@ -1,109 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 1, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_minus_rla" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.8917534983307565, - "train_loss": 0.6392366142168532, - "val_score": -1.7708043114771035 - }, - { - "epoch": 1, - "test_score": -1.8698356715330888, - "train_loss": 0.5946752181018355, - "val_score": -1.7504887579540844 - }, - { - "epoch": 2, - "test_score": -1.9423982169881553, - "train_loss": 0.5839949937632484, - "val_score": -1.8282106167560017 - }, - { - "epoch": 3, - "test_score": -1.9397289621633413, - "train_loss": 0.5700914755354832, - "val_score": -1.8241841151607034 - }, - { - "epoch": 4, - "test_score": -1.9145957982346236, - "train_loss": 0.5711912164722917, - "val_score": -1.8062533896655315 - }, - { - "epoch": 5, - "test_score": -1.8782919912631686, - "train_loss": 0.5749758856139914, - "val_score": -1.7671069787094078 - }, - { - "epoch": 6, - "test_score": -1.9406991204252366, - "train_loss": 0.5706792591262038, - "val_score": -1.8289964343245315 - }, - { - "epoch": 7, - "test_score": -1.92082874383054, - "train_loss": 0.5622948334164863, - "val_score": -1.805523699291935 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.8698356715330888, - "score": -1.8698356715330888 - }, - "val": { - "rmse": 1.7504887579540844, - "score": -1.7504887579540844 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 37.817176818847656, - "variant": "all_minus_rla" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2.toml deleted file mode 100644 index 16b347221..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2.toml +++ /dev/null @@ -1,32 +0,0 @@ -dataset = "delivery-eta" -variant = "all_minus_rla" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2/report.json deleted file mode 100644 index 8bf5ea3bd..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2/report.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_minus_rla" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.9066122992751606, - "train_loss": 0.6281222657565653, - "val_score": -1.7852811206241392 - }, - { - "epoch": 1, - "test_score": -1.9025621230099772, - "train_loss": 0.5781265331880889, - "val_score": -1.7918848156269322 - }, - { - "epoch": 2, - "test_score": -1.932680473043221, - "train_loss": 0.5821000093961284, - "val_score": -1.8283426530172184 - }, - { - "epoch": 3, - "test_score": -1.923708591523656, - "train_loss": 0.5710510563676374, - "val_score": -1.80417952881678 - }, - { - "epoch": 4, - "test_score": -1.9435592165228985, - "train_loss": 0.5663256745268829, - "val_score": -1.8293261372329226 - }, - { - "epoch": 5, - "test_score": -1.905806505486743, - "train_loss": 0.5734849132760598, - "val_score": -1.7990136729087136 - }, - { - "epoch": 6, - "test_score": -1.8987611722334956, - "train_loss": 0.5716026260034881, - "val_score": -1.7859538026668018 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.9066122992751606, - "score": -1.9066122992751606 - }, - "val": { - "rmse": 1.7852811206241392, - "score": -1.7852811206241392 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/all_minus_rla-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 32.113162994384766, - "variant": "all_minus_rla" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0.toml deleted file mode 100644 index 4dc608f3b..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0.toml +++ /dev/null @@ -1,27 +0,0 @@ -dataset = "delivery-eta" -variant = "baseline_plr" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0/report.json deleted file mode 100644 index 327d30f92..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0/report.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "baseline_plr" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.8396023796480883, - "train_loss": 0.6278780659619909, - "val_score": -1.7395649830960092 - }, - { - "epoch": 1, - "test_score": -1.889375825680506, - "train_loss": 0.5447326698442445, - "val_score": -1.7913296922311428 - }, - { - "epoch": 2, - "test_score": -1.9264100694336173, - "train_loss": 0.5396799610479035, - "val_score": -1.8183812059183937 - }, - { - "epoch": 3, - "test_score": -1.9165897423055016, - "train_loss": 0.53633236558768, - "val_score": -1.8094179981768903 - }, - { - "epoch": 4, - "test_score": -1.9316801215628367, - "train_loss": 0.5344857908513424, - "val_score": -1.823777596759711 - }, - { - "epoch": 5, - "test_score": -1.9056037057379331, - "train_loss": 0.5330761410024044, - "val_score": -1.797133319365044 - }, - { - "epoch": 6, - "test_score": -1.9031948559097036, - "train_loss": 0.5317075433087175, - "val_score": -1.802448750162817 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.8396023796480883, - "score": -1.8396023796480883 - }, - "val": { - "rmse": 1.7395649830960092, - "score": -1.7395649830960092 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 22.571192026138306, - "variant": "baseline_plr" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1.toml deleted file mode 100644 index 4eb0c2a12..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1.toml +++ /dev/null @@ -1,27 +0,0 @@ -dataset = "delivery-eta" -variant = "baseline_plr" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1/report.json deleted file mode 100644 index eb6514129..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1/report.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "baseline_plr" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.8639852605326992, - "train_loss": 0.6245106560905485, - "val_score": -1.7606024408977228 - }, - { - "epoch": 1, - "test_score": -1.8734272400588596, - "train_loss": 0.5430235405908014, - "val_score": -1.7644821113940043 - }, - { - "epoch": 2, - "test_score": -1.946784936771937, - "train_loss": 0.5393460347269573, - "val_score": -1.8353593930658616 - }, - { - "epoch": 3, - "test_score": -1.9135835009303772, - "train_loss": 0.5361843874854763, - "val_score": -1.8047017679022026 - }, - { - "epoch": 4, - "test_score": -1.8845223541593812, - "train_loss": 0.5334207896333542, - "val_score": -1.7767816010897883 - }, - { - "epoch": 5, - "test_score": -1.9256172032927374, - "train_loss": 0.5318533973972293, - "val_score": -1.8147362692408484 - }, - { - "epoch": 6, - "test_score": -1.889762240193959, - "train_loss": 0.530044365755833, - "val_score": -1.78181903517071 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.8639852605326992, - "score": -1.8639852605326992 - }, - "val": { - "rmse": 1.7606024408977228, - "score": -1.7606024408977228 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 23.594813346862793, - "variant": "baseline_plr" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2.toml deleted file mode 100644 index bc63f8e11..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2.toml +++ /dev/null @@ -1,27 +0,0 @@ -dataset = "delivery-eta" -variant = "baseline_plr" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2/report.json deleted file mode 100644 index 5ebb516e5..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2/report.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 4, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "baseline_plr" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.8721481888104075, - "train_loss": 0.6234128196744153, - "val_score": -1.7699674022496135 - }, - { - "epoch": 1, - "test_score": -1.873120446694421, - "train_loss": 0.5432704556597411, - "val_score": -1.773794062727797 - }, - { - "epoch": 2, - "test_score": -1.9151929358757613, - "train_loss": 0.5391251908166566, - "val_score": -1.815582285393361 - }, - { - "epoch": 3, - "test_score": -1.9287470328900895, - "train_loss": 0.5359159618833639, - "val_score": -1.8178189598311036 - }, - { - "epoch": 4, - "test_score": -1.8590972436481992, - "train_loss": 0.5342420942156855, - "val_score": -1.762300314988203 - }, - { - "epoch": 5, - "test_score": -1.8938041628907152, - "train_loss": 0.5325591675121418, - "val_score": -1.7882742665405706 - }, - { - "epoch": 6, - "test_score": -1.9157789980547906, - "train_loss": 0.5309514068338993, - "val_score": -1.8092274554615024 - }, - { - "epoch": 7, - "test_score": -1.9134677507899076, - "train_loss": 0.5297948411346352, - "val_score": -1.8133076151024474 - }, - { - "epoch": 8, - "test_score": -1.944729330749166, - "train_loss": 0.5287812985207913, - "val_score": -1.8367321486087818 - }, - { - "epoch": 9, - "test_score": -1.8971487309694048, - "train_loss": 0.5279093323832881, - "val_score": -1.7899461587874737 - }, - { - "epoch": 10, - "test_score": -1.909957469746283, - "train_loss": 0.5260672956487559, - "val_score": -1.8055805457252359 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.8590972436481992, - "score": -1.8590972436481992 - }, - "val": { - "rmse": 1.762300314988203, - "score": -1.762300314988203 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/baseline_plr-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 36.959254026412964, - "variant": "baseline_plr" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0.toml deleted file mode 100644 index e3acfbe87..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0.toml +++ /dev/null @@ -1,28 +0,0 @@ -dataset = "delivery-eta" -variant = "cf_fisd_only" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0/report.json deleted file mode 100644 index 61b118e50..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0/report.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "cf_fisd_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.847462746808535, - "train_loss": 0.5944465786001109, - "val_score": -1.7329661515326225 - }, - { - "epoch": 1, - "test_score": -1.8863803093999616, - "train_loss": 0.5455469045325787, - "val_score": -1.782074921309916 - }, - { - "epoch": 2, - "test_score": -1.9238514235581599, - "train_loss": 0.5399144372365771, - "val_score": -1.8125378177249112 - }, - { - "epoch": 3, - "test_score": -1.9124455063978703, - "train_loss": 0.53612889512612, - "val_score": -1.7996051779172895 - }, - { - "epoch": 4, - "test_score": -1.9290277998222791, - "train_loss": 0.5339680866603433, - "val_score": -1.8197581220598342 - }, - { - "epoch": 5, - "test_score": -1.9045877557287432, - "train_loss": 0.5323602639845688, - "val_score": -1.7948221172717103 - }, - { - "epoch": 6, - "test_score": -1.9236647173423764, - "train_loss": 0.5312561038201743, - "val_score": -1.8180548948722761 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.847462746808535, - "score": -1.847462746808535 - }, - "val": { - "rmse": 1.7329661515326225, - "score": -1.7329661515326225 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 21.50657367706299, - "variant": "cf_fisd_only" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1.toml deleted file mode 100644 index 763e4dbdd..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1.toml +++ /dev/null @@ -1,28 +0,0 @@ -dataset = "delivery-eta" -variant = "cf_fisd_only" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1/report.json deleted file mode 100644 index 7bfb46dde..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1/report.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 1, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "cf_fisd_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.8773772110895843, - "train_loss": 0.592602454180265, - "val_score": -1.7619405467279132 - }, - { - "epoch": 1, - "test_score": -1.8568220804711988, - "train_loss": 0.5443947880807585, - "val_score": -1.7414478733432992 - }, - { - "epoch": 2, - "test_score": -1.9585844717578214, - "train_loss": 0.5396959011572121, - "val_score": -1.8442746078795882 - }, - { - "epoch": 3, - "test_score": -1.9227801416713162, - "train_loss": 0.5361053360639697, - "val_score": -1.8145431315556086 - }, - { - "epoch": 4, - "test_score": -1.8821978693611068, - "train_loss": 0.5335870689719263, - "val_score": -1.77016809721336 - }, - { - "epoch": 5, - "test_score": -1.9337962034952991, - "train_loss": 0.5318883611773052, - "val_score": -1.8237204677813132 - }, - { - "epoch": 6, - "test_score": -1.9025955816813702, - "train_loss": 0.530138851082238, - "val_score": -1.7919569963123252 - }, - { - "epoch": 7, - "test_score": -1.9315667519170043, - "train_loss": 0.5293886254738717, - "val_score": -1.8192720509490548 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.8568220804711988, - "score": -1.8568220804711988 - }, - "val": { - "rmse": 1.7414478733432992, - "score": -1.7414478733432992 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 27.340318202972412, - "variant": "cf_fisd_only" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2.toml deleted file mode 100644 index 0efb09181..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2.toml +++ /dev/null @@ -1,28 +0,0 @@ -dataset = "delivery-eta" -variant = "cf_fisd_only" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2/report.json deleted file mode 100644 index a5b7220ef..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2/report.json +++ /dev/null @@ -1,123 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 4, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "cf_fisd_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.9230385956246019, - "train_loss": 0.5934791673709007, - "val_score": -1.8042379372536939 - }, - { - "epoch": 1, - "test_score": -1.8948157028119332, - "train_loss": 0.5450501676893582, - "val_score": -1.7880606699001504 - }, - { - "epoch": 2, - "test_score": -1.9230178908310813, - "train_loss": 0.5397137101114231, - "val_score": -1.818952191312291 - }, - { - "epoch": 3, - "test_score": -1.896522906586717, - "train_loss": 0.536448588118936, - "val_score": -1.7856421946551229 - }, - { - "epoch": 4, - "test_score": -1.8737455622217765, - "train_loss": 0.5346145312281421, - "val_score": -1.774137585629917 - }, - { - "epoch": 5, - "test_score": -1.9030765946970747, - "train_loss": 0.5325972875104333, - "val_score": -1.7950323857438173 - }, - { - "epoch": 6, - "test_score": -1.9139981005987072, - "train_loss": 0.5305591375723372, - "val_score": -1.8049593634006922 - }, - { - "epoch": 7, - "test_score": -1.907357628885035, - "train_loss": 0.5295915420908127, - "val_score": -1.8045180600373205 - }, - { - "epoch": 8, - "test_score": -1.9360437150070906, - "train_loss": 0.5289334767926348, - "val_score": -1.8271244277885352 - }, - { - "epoch": 9, - "test_score": -1.913430744154855, - "train_loss": 0.5275534989624998, - "val_score": -1.800273236093376 - }, - { - "epoch": 10, - "test_score": -1.9140615035073936, - "train_loss": 0.5261308607393808, - "val_score": -1.8077536293935552 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.8737455622217765, - "score": -1.8737455622217765 - }, - "val": { - "rmse": 1.774137585629917, - "score": -1.774137585629917 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/cf_fisd_only-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 36.74504613876343, - "variant": "cf_fisd_only" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0.toml deleted file mode 100644 index a7f0b4166..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "delivery-eta" -variant = "esam_only" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0/report.json deleted file mode 100644 index 2e7ff74b5..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0/report.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "esam_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.8414060273006467, - "train_loss": 0.6287253525135291, - "val_score": -1.7408771462387886 - }, - { - "epoch": 1, - "test_score": -1.8874872826154119, - "train_loss": 0.5451243132570364, - "val_score": -1.7892239426978986 - }, - { - "epoch": 2, - "test_score": -1.9253018229293297, - "train_loss": 0.5400790667881931, - "val_score": -1.817427678197544 - }, - { - "epoch": 3, - "test_score": -1.915089234546431, - "train_loss": 0.536670585618402, - "val_score": -1.8080386139915399 - }, - { - "epoch": 4, - "test_score": -1.9321186025873545, - "train_loss": 0.5349389023154322, - "val_score": -1.823938711755445 - }, - { - "epoch": 5, - "test_score": -1.905402698756221, - "train_loss": 0.5335488456444148, - "val_score": -1.7974111669286685 - }, - { - "epoch": 6, - "test_score": -1.9018297068818415, - "train_loss": 0.5321317349907255, - "val_score": -1.8006889667158776 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.8414060273006467, - "score": -1.8414060273006467 - }, - "val": { - "rmse": 1.7408771462387886, - "score": -1.7408771462387886 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 31.279943466186523, - "variant": "esam_only" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1.toml deleted file mode 100644 index c42ffbc0f..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "delivery-eta" -variant = "esam_only" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1/report.json deleted file mode 100644 index ba56e807a..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1/report.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "esam_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.867077237628412, - "train_loss": 0.6253672166027292, - "val_score": -1.7633451748518263 - }, - { - "epoch": 1, - "test_score": -1.8728420555294343, - "train_loss": 0.5435903977303609, - "val_score": -1.7639967575865203 - }, - { - "epoch": 2, - "test_score": -1.9459718865915152, - "train_loss": 0.5396962204988855, - "val_score": -1.8349061057367806 - }, - { - "epoch": 3, - "test_score": -1.9133081311992928, - "train_loss": 0.5365906061917326, - "val_score": -1.8041447075027965 - }, - { - "epoch": 4, - "test_score": -1.8830463058482847, - "train_loss": 0.5339189865728364, - "val_score": -1.7755051713340173 - }, - { - "epoch": 5, - "test_score": -1.925044725567323, - "train_loss": 0.5321700340204866, - "val_score": -1.814003090949406 - }, - { - "epoch": 6, - "test_score": -1.8887670546240742, - "train_loss": 0.5302761634335901, - "val_score": -1.7808658788001235 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.867077237628412, - "score": -1.867077237628412 - }, - "val": { - "rmse": 1.7633451748518263, - "score": -1.7633451748518263 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 33.070332765579224, - "variant": "esam_only" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2.toml deleted file mode 100644 index f5ee21155..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "delivery-eta" -variant = "esam_only" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2/report.json deleted file mode 100644 index 7dd64afb4..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2/report.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 4, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "esam_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.8725806203887048, - "train_loss": 0.6242489044683693, - "val_score": -1.7706746497769004 - }, - { - "epoch": 1, - "test_score": -1.8729241005732664, - "train_loss": 0.5437626016401026, - "val_score": -1.7736281237893647 - }, - { - "epoch": 2, - "test_score": -1.9148552945994042, - "train_loss": 0.5395388853375929, - "val_score": -1.815533434451519 - }, - { - "epoch": 3, - "test_score": -1.926554681246965, - "train_loss": 0.5361864662518466, - "val_score": -1.815812865105869 - }, - { - "epoch": 4, - "test_score": -1.8608628418981936, - "train_loss": 0.5344241089629431, - "val_score": -1.7636637639971298 - }, - { - "epoch": 5, - "test_score": -1.8918575336215864, - "train_loss": 0.5327327956248374, - "val_score": -1.7864739627729052 - }, - { - "epoch": 6, - "test_score": -1.9155521746136261, - "train_loss": 0.5312667256289155, - "val_score": -1.8085834016771645 - }, - { - "epoch": 7, - "test_score": -1.9122361792814149, - "train_loss": 0.529884976844718, - "val_score": -1.812130528643339 - }, - { - "epoch": 8, - "test_score": -1.9455652034502635, - "train_loss": 0.528975662958883, - "val_score": -1.8374890203861554 - }, - { - "epoch": 9, - "test_score": -1.8964897807836834, - "train_loss": 0.5281089191889241, - "val_score": -1.7885813503347106 - }, - { - "epoch": 10, - "test_score": -1.9095956802829104, - "train_loss": 0.5263300951379929, - "val_score": -1.8048084433187321 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.8608628418981936, - "score": -1.8608628418981936 - }, - "val": { - "rmse": 1.7636637639971298, - "score": -1.7636637639971298 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/esam_only-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 53.07820463180542, - "variant": "esam_only" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0.toml deleted file mode 100644 index a05013205..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "delivery-eta" -variant = "mfb_only" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0/report.json deleted file mode 100644 index 41799d2f2..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0/report.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "mfb_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.882313199140896, - "train_loss": 0.6657763436762956, - "val_score": -1.7760818193507641 - }, - { - "epoch": 1, - "test_score": -1.9108278307008544, - "train_loss": 0.5919238420298499, - "val_score": -1.7950738919278724 - }, - { - "epoch": 2, - "test_score": -1.8997331758164053, - "train_loss": 0.5805986815125402, - "val_score": -1.7929712765079557 - }, - { - "epoch": 3, - "test_score": -1.9036324461863272, - "train_loss": 0.5849480929165861, - "val_score": -1.7954276848902972 - }, - { - "epoch": 4, - "test_score": -1.9270621908591754, - "train_loss": 0.5751130467783796, - "val_score": -1.8031489431817904 - }, - { - "epoch": 5, - "test_score": -1.9185303528159003, - "train_loss": 0.5802982014461155, - "val_score": -1.8075533484134956 - }, - { - "epoch": 6, - "test_score": -1.904226260709987, - "train_loss": 0.5747161829558602, - "val_score": -1.796530184625187 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.882313199140896, - "score": -1.882313199140896 - }, - "val": { - "rmse": 1.7760818193507641, - "score": -1.7760818193507641 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 22.597419023513794, - "variant": "mfb_only" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1.toml deleted file mode 100644 index 4c6383fb7..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "delivery-eta" -variant = "mfb_only" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1/report.json deleted file mode 100644 index 0b08e793a..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1/report.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 4, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "mfb_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.9317965700093218, - "train_loss": 0.6622442111481716, - "val_score": -1.8175719099760428 - }, - { - "epoch": 1, - "test_score": -1.9051803339309807, - "train_loss": 0.5844372032332594, - "val_score": -1.7967930650689325 - }, - { - "epoch": 2, - "test_score": -1.9433397455038781, - "train_loss": 0.5849274127152715, - "val_score": -1.8242998452305552 - }, - { - "epoch": 3, - "test_score": -1.9411426265421559, - "train_loss": 0.5797237166523064, - "val_score": -1.8226949139094244 - }, - { - "epoch": 4, - "test_score": -1.8999776362056646, - "train_loss": 0.573150089622414, - "val_score": -1.7831396152179502 - }, - { - "epoch": 5, - "test_score": -1.9150731746547511, - "train_loss": 0.5799409951606807, - "val_score": -1.7980624048862248 - }, - { - "epoch": 6, - "test_score": -1.9013377198618562, - "train_loss": 0.5672564837184265, - "val_score": -1.783942212533094 - }, - { - "epoch": 7, - "test_score": -1.9636375410585374, - "train_loss": 0.579314707839576, - "val_score": -1.8430848942503817 - }, - { - "epoch": 8, - "test_score": -1.930902137414017, - "train_loss": 0.5656810683490586, - "val_score": -1.8118882297797387 - }, - { - "epoch": 9, - "test_score": -1.9378414468459522, - "train_loss": 0.567348591820167, - "val_score": -1.8191206144805452 - }, - { - "epoch": 10, - "test_score": -1.9203693105241764, - "train_loss": 0.5712279420699516, - "val_score": -1.8070823335663142 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.8999776362056646, - "score": -1.8999776362056646 - }, - "val": { - "rmse": 1.7831396152179502, - "score": -1.7831396152179502 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 37.64791131019592, - "variant": "mfb_only" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2.toml deleted file mode 100644 index 952dde1cc..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "delivery-eta" -variant = "mfb_only" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2/report.json deleted file mode 100644 index 809a7c6bb..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2/report.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 6, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "mfb_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.9128695137240894, - "train_loss": 0.665501472723745, - "val_score": -1.7964501583575907 - }, - { - "epoch": 1, - "test_score": -1.9440009050249418, - "train_loss": 0.5934047594557713, - "val_score": -1.8344874073011204 - }, - { - "epoch": 2, - "test_score": -1.9288621133395232, - "train_loss": 0.5766440614731643, - "val_score": -1.8188880946637216 - }, - { - "epoch": 3, - "test_score": -1.925762555571344, - "train_loss": 0.5732441340049688, - "val_score": -1.8097932269607562 - }, - { - "epoch": 4, - "test_score": -1.9167776974017394, - "train_loss": 0.586352648961283, - "val_score": -1.8035885986741023 - }, - { - "epoch": 5, - "test_score": -1.926642358861961, - "train_loss": 0.5883107520367977, - "val_score": -1.8093135711309383 - }, - { - "epoch": 6, - "test_score": -1.8822192131865638, - "train_loss": 0.5684957961096381, - "val_score": -1.764251917763006 - }, - { - "epoch": 7, - "test_score": -1.928180183103219, - "train_loss": 0.5747323318989608, - "val_score": -1.8243644705165025 - }, - { - "epoch": 8, - "test_score": -1.9409041494201429, - "train_loss": 0.5614554407822825, - "val_score": -1.8332152617609774 - }, - { - "epoch": 9, - "test_score": -1.9376158525839038, - "train_loss": 0.5655314191414492, - "val_score": -1.820973681432002 - }, - { - "epoch": 10, - "test_score": -1.8920766758614376, - "train_loss": 0.5720564540285263, - "val_score": -1.7866258310993834 - }, - { - "epoch": 11, - "test_score": -1.923358499746884, - "train_loss": 0.5571396168131028, - "val_score": -1.812032902479919 - }, - { - "epoch": 12, - "test_score": -1.9155238586907501, - "train_loss": 0.5610200036181151, - "val_score": -1.8033300141088338 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.8822192131865638, - "score": -1.8822192131865638 - }, - "val": { - "rmse": 1.764251917763006, - "score": -1.764251917763006 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/mfb_only-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 42.991671562194824, - "variant": "mfb_only" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0.toml deleted file mode 100644 index 2daabeb51..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0.toml +++ /dev/null @@ -1,30 +0,0 @@ -dataset = "delivery-eta" -variant = "rla_only" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0/report.json deleted file mode 100644 index 2fe61d66d..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0/report.json +++ /dev/null @@ -1,161 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 10, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "rla_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.8991126598508863, - "train_loss": 0.6279589468545287, - "val_score": -1.7893601882007086 - }, - { - "epoch": 1, - "test_score": -1.9124187651691174, - "train_loss": 0.5441900978993325, - "val_score": -1.8123750972462311 - }, - { - "epoch": 2, - "test_score": -1.8848644171404472, - "train_loss": 0.5390364954071324, - "val_score": -1.7817099797140792 - }, - { - "epoch": 3, - "test_score": -1.9074590007130547, - "train_loss": 0.536344505139511, - "val_score": -1.7976066095095695 - }, - { - "epoch": 4, - "test_score": -1.8838248162957778, - "train_loss": 0.5340189500881808, - "val_score": -1.7804151887691884 - }, - { - "epoch": 5, - "test_score": -1.9015802813586458, - "train_loss": 0.532935598688404, - "val_score": -1.796179462659246 - }, - { - "epoch": 6, - "test_score": -1.9064965635839466, - "train_loss": 0.5309413510517482, - "val_score": -1.8006959179170425 - }, - { - "epoch": 7, - "test_score": -1.9036021995153383, - "train_loss": 0.5296575087700447, - "val_score": -1.7933113252903616 - }, - { - "epoch": 8, - "test_score": -1.898886042851269, - "train_loss": 0.5284191111143488, - "val_score": -1.792912966456737 - }, - { - "epoch": 9, - "test_score": -1.8955831537303498, - "train_loss": 0.5278057240656693, - "val_score": -1.793970164966326 - }, - { - "epoch": 10, - "test_score": -1.8738282038863792, - "train_loss": 0.526295984530971, - "val_score": -1.7673837460024195 - }, - { - "epoch": 11, - "test_score": -1.9061960931879003, - "train_loss": 0.5257948676165003, - "val_score": -1.8011830309901071 - }, - { - "epoch": 12, - "test_score": -1.9085130234899337, - "train_loss": 0.5251704580592413, - "val_score": -1.8059021136263849 - }, - { - "epoch": 13, - "test_score": -1.9503421947808426, - "train_loss": 0.5246895671325879, - "val_score": -1.8436843731078842 - }, - { - "epoch": 14, - "test_score": -1.9205415643406698, - "train_loss": 0.5239885138334149, - "val_score": -1.809490599423118 - }, - { - "epoch": 15, - "test_score": -1.9227169642652717, - "train_loss": 0.5230375030179963, - "val_score": -1.8190888315355058 - }, - { - "epoch": 16, - "test_score": -1.90566319673516, - "train_loss": 0.5221649397463695, - "val_score": -1.7955265458730523 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.8738282038863792, - "score": -1.8738282038863792 - }, - "val": { - "rmse": 1.7673837460024195, - "score": -1.7673837460024195 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 59.403921604156494, - "variant": "rla_only" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1.toml deleted file mode 100644 index bd2c0266b..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1.toml +++ /dev/null @@ -1,30 +0,0 @@ -dataset = "delivery-eta" -variant = "rla_only" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1/report.json deleted file mode 100644 index f59b665db..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1/report.json +++ /dev/null @@ -1,113 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 2, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "rla_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.9529116094366537, - "train_loss": 0.626599464973394, - "val_score": -1.854167409603813 - }, - { - "epoch": 1, - "test_score": -1.9045921996561277, - "train_loss": 0.5439584403142442, - "val_score": -1.8019199039978397 - }, - { - "epoch": 2, - "test_score": -1.8829804025076913, - "train_loss": 0.5388047882240184, - "val_score": -1.7775892151066777 - }, - { - "epoch": 3, - "test_score": -1.917166050524368, - "train_loss": 0.5356628246551013, - "val_score": -1.8086206421799282 - }, - { - "epoch": 4, - "test_score": -1.8981720538645561, - "train_loss": 0.533457537854675, - "val_score": -1.7923243742261243 - }, - { - "epoch": 5, - "test_score": -1.9191541562961174, - "train_loss": 0.5313687770471086, - "val_score": -1.8099570357771626 - }, - { - "epoch": 6, - "test_score": -1.9167533177677576, - "train_loss": 0.530146632533874, - "val_score": -1.8146295205102208 - }, - { - "epoch": 7, - "test_score": -1.92939379229104, - "train_loss": 0.5287531240143045, - "val_score": -1.8220407034371349 - }, - { - "epoch": 8, - "test_score": -1.9524148489228184, - "train_loss": 0.5275940823293951, - "val_score": -1.8425391795781818 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.8829804025076913, - "score": -1.8829804025076913 - }, - "val": { - "rmse": 1.7775892151066777, - "score": -1.7775892151066777 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 32.074143171310425, - "variant": "rla_only" -} diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2.toml deleted file mode 100644 index b3419b31e..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2.toml +++ /dev/null @@ -1,30 +0,0 @@ -dataset = "delivery-eta" -variant = "rla_only" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/delivery-eta" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2/report.json deleted file mode 100644 index f3409a89f..000000000 --- a/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2/report.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/delivery-eta" - }, - "dataset": "delivery-eta", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "rla_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2.toml", - "dataset": "delivery-eta", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -1.88058796129553, - "train_loss": 0.6275782894044026, - "val_score": -1.7734446252352045 - }, - { - "epoch": 1, - "test_score": -1.9163067797704876, - "train_loss": 0.5442314702663978, - "val_score": -1.8092503849001196 - }, - { - "epoch": 2, - "test_score": -1.9333606926308549, - "train_loss": 0.5406341039351303, - "val_score": -1.8277433600207929 - }, - { - "epoch": 3, - "test_score": -1.892036415576302, - "train_loss": 0.5362807738519934, - "val_score": -1.7886593960062 - }, - { - "epoch": 4, - "test_score": -1.931961140978318, - "train_loss": 0.5344498851438508, - "val_score": -1.8238907382305147 - }, - { - "epoch": 5, - "test_score": -1.8943497718331832, - "train_loss": 0.5325871710359615, - "val_score": -1.7891810348755857 - }, - { - "epoch": 6, - "test_score": -1.9185911206002868, - "train_loss": 0.5309783061490442, - "val_score": -1.8074469009556464 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 1.88058796129553, - "score": -1.88058796129553 - }, - "val": { - "rmse": 1.7734446252352045, - "score": -1.7734446252352045 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/delivery-eta/rla_only-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 26.989912509918213, - "variant": "rla_only" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0.toml deleted file mode 100644 index c91474fcb..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0.toml +++ /dev/null @@ -1,35 +0,0 @@ -dataset = "ecom-offers" -variant = "all_four_combined" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0/report.json deleted file mode 100644 index 242fa0a59..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0/report.json +++ /dev/null @@ -1,184 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 19, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_four_combined" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5681271222366269, - "train_loss": 0.5521719566097966, - "val_score": 0.5900086445530405 - }, - { - "epoch": 1, - "test_score": 0.5666844169705509, - "train_loss": 0.5123001651631461, - "val_score": 0.5863801419866583 - }, - { - "epoch": 2, - "test_score": 0.5681515585303227, - "train_loss": 0.5097376847708667, - "val_score": 0.588203584427758 - }, - { - "epoch": 3, - "test_score": 0.5695960057036864, - "train_loss": 0.507381233352202, - "val_score": 0.5914278861087139 - }, - { - "epoch": 4, - "test_score": 0.5722711537685954, - "train_loss": 0.5049635193966053, - "val_score": 0.5967446024280252 - }, - { - "epoch": 5, - "test_score": 0.5690483835267741, - "train_loss": 0.5053603251775106, - "val_score": 0.5903752258153991 - }, - { - "epoch": 6, - "test_score": 0.5708569203180814, - "train_loss": 0.5041680694730194, - "val_score": 0.5948900154498427 - }, - { - "epoch": 7, - "test_score": 0.5712752012296551, - "train_loss": 0.5012796890956385, - "val_score": 0.5947254295989219 - }, - { - "epoch": 8, - "test_score": 0.5715983591258145, - "train_loss": 0.5017772890903331, - "val_score": 0.5956531174159587 - }, - { - "epoch": 9, - "test_score": 0.5696216228989421, - "train_loss": 0.5025770465532938, - "val_score": 0.5918668799860465 - }, - { - "epoch": 10, - "test_score": 0.5728374410090079, - "train_loss": 0.5002644200015951, - "val_score": 0.5988367627801356 - }, - { - "epoch": 11, - "test_score": 0.5718931908819324, - "train_loss": 0.5004737333015159, - "val_score": 0.5952545992753181 - }, - { - "epoch": 12, - "test_score": 0.5711061773326337, - "train_loss": 0.4999077397364157, - "val_score": 0.593878350521634 - }, - { - "epoch": 13, - "test_score": 0.5740794395020907, - "train_loss": 0.49866839636255195, - "val_score": 0.5986739163945447 - }, - { - "epoch": 14, - "test_score": 0.5721571115323896, - "train_loss": 0.5007636806479207, - "val_score": 0.5954374826799741 - }, - { - "epoch": 15, - "test_score": 0.5738696419567221, - "train_loss": 0.4993751821694551, - "val_score": 0.5991782286691394 - }, - { - "epoch": 16, - "test_score": 0.5739100870602782, - "train_loss": 0.4982121962088126, - "val_score": 0.5991561942131909 - }, - { - "epoch": 17, - "test_score": 0.5756130369945278, - "train_loss": 0.4973638112898226, - "val_score": 0.6010971648086056 - }, - { - "epoch": 18, - "test_score": 0.5745054846089597, - "train_loss": 0.496809223735774, - "val_score": 0.5997656556928566 - }, - { - "epoch": 19, - "test_score": 0.575590289916974, - "train_loss": 0.49563960841408483, - "val_score": 0.6024052943310221 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.575590289916974, - "score": 0.575590289916974 - }, - "val": { - "roc-auc": 0.6024052943310221, - "score": 0.6024052943310221 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/0", - "seed": 0, - "task_type": "binclass", - "time_seconds": 35.4436354637146, - "variant": "all_four_combined" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1.toml deleted file mode 100644 index 48180f642..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1.toml +++ /dev/null @@ -1,35 +0,0 @@ -dataset = "ecom-offers" -variant = "all_four_combined" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1/report.json deleted file mode 100644 index c664ae19a..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1/report.json +++ /dev/null @@ -1,184 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 17, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_four_combined" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.570155176539948, - "train_loss": 0.5530174418731972, - "val_score": 0.5931137338939699 - }, - { - "epoch": 1, - "test_score": 0.5710579773322195, - "train_loss": 0.5131659612611488, - "val_score": 0.5945566940913031 - }, - { - "epoch": 2, - "test_score": 0.5710214871640712, - "train_loss": 0.5079680477027539, - "val_score": 0.5940891611895743 - }, - { - "epoch": 3, - "test_score": 0.5690187556054855, - "train_loss": 0.5082374629047182, - "val_score": 0.5892378108109451 - }, - { - "epoch": 4, - "test_score": 0.5692918414158183, - "train_loss": 0.5057879333142881, - "val_score": 0.5899993280946632 - }, - { - "epoch": 5, - "test_score": 0.5717230294760591, - "train_loss": 0.5025530799671456, - "val_score": 0.5962786947470815 - }, - { - "epoch": 6, - "test_score": 0.5718020382993217, - "train_loss": 0.5034321209898701, - "val_score": 0.59612584660183 - }, - { - "epoch": 7, - "test_score": 0.5729451634072069, - "train_loss": 0.5040367698227918, - "val_score": 0.5965978682085837 - }, - { - "epoch": 8, - "test_score": 0.5713202769550761, - "train_loss": 0.5016454839044147, - "val_score": 0.5942215448200439 - }, - { - "epoch": 9, - "test_score": 0.5705815532910871, - "train_loss": 0.5025773329867257, - "val_score": 0.5937047577781998 - }, - { - "epoch": 10, - "test_score": 0.572931395520779, - "train_loss": 0.5005058971820054, - "val_score": 0.5959005858416042 - }, - { - "epoch": 11, - "test_score": 0.5737611477882123, - "train_loss": 0.49932017315317084, - "val_score": 0.5962125066171546 - }, - { - "epoch": 12, - "test_score": 0.5760470260664834, - "train_loss": 0.4990245197658186, - "val_score": 0.5994599151786585 - }, - { - "epoch": 13, - "test_score": 0.5736891096935801, - "train_loss": 0.4994683022852297, - "val_score": 0.5968734702747561 - }, - { - "epoch": 14, - "test_score": 0.5759794977140287, - "train_loss": 0.4970110097417125, - "val_score": 0.6010394307750069 - }, - { - "epoch": 15, - "test_score": 0.5753586999336326, - "train_loss": 0.49861614792435255, - "val_score": 0.5993183809286673 - }, - { - "epoch": 16, - "test_score": 0.5749285542158353, - "train_loss": 0.4977775944603814, - "val_score": 0.5981364732019077 - }, - { - "epoch": 17, - "test_score": 0.5770100559377882, - "train_loss": 0.4978238934720004, - "val_score": 0.6032831567852186 - }, - { - "epoch": 18, - "test_score": 0.575521803825459, - "train_loss": 0.4963515991414035, - "val_score": 0.6008931203659741 - }, - { - "epoch": 19, - "test_score": 0.5767410428430715, - "train_loss": 0.4981405012033604, - "val_score": 0.6010097161373898 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5770100559377882, - "score": 0.5770100559377882 - }, - "val": { - "roc-auc": 0.6032831567852186, - "score": 0.6032831567852186 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/1", - "seed": 1, - "task_type": "binclass", - "time_seconds": 36.04194712638855, - "variant": "all_four_combined" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2.toml deleted file mode 100644 index 95d814f71..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2.toml +++ /dev/null @@ -1,35 +0,0 @@ -dataset = "ecom-offers" -variant = "all_four_combined" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2/report.json deleted file mode 100644 index d3c459911..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2/report.json +++ /dev/null @@ -1,112 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 1, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_four_combined" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5662744736579172, - "train_loss": 0.5533291416035758, - "val_score": 0.5870080889707623 - }, - { - "epoch": 1, - "test_score": 0.5734697781503159, - "train_loss": 0.5128997222140983, - "val_score": 0.5995627021013641 - }, - { - "epoch": 2, - "test_score": 0.5708813690096939, - "train_loss": 0.5101203283777943, - "val_score": 0.5947186744295367 - }, - { - "epoch": 3, - "test_score": 0.572774698252376, - "train_loss": 0.5073363532622656, - "val_score": 0.5972117999667349 - }, - { - "epoch": 4, - "test_score": 0.569280137782511, - "train_loss": 0.5072464628352059, - "val_score": 0.5914310996972048 - }, - { - "epoch": 5, - "test_score": 0.568470256275971, - "train_loss": 0.5035725958921291, - "val_score": 0.5907585457462503 - }, - { - "epoch": 6, - "test_score": 0.570999490160474, - "train_loss": 0.5035661116794303, - "val_score": 0.5938800347073453 - }, - { - "epoch": 7, - "test_score": 0.5664802511788156, - "train_loss": 0.5036690522123266, - "val_score": 0.5869948255476216 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5734697781503159, - "score": 0.5734697781503159 - }, - "val": { - "roc-auc": 0.5995627021013641, - "score": 0.5995627021013641 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_four_combined-evaluation/2", - "seed": 2, - "task_type": "binclass", - "time_seconds": 15.193928480148315, - "variant": "all_four_combined" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0.toml deleted file mode 100644 index df11e7b0d..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0.toml +++ /dev/null @@ -1,34 +0,0 @@ -dataset = "ecom-offers" -variant = "all_minus_cf_fisd" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0/report.json deleted file mode 100644 index 79a9171e3..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0/report.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_minus_cf_fisd" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.574504443183962, - "train_loss": 0.5605815064024042, - "val_score": 0.6001605290639818 - }, - { - "epoch": 1, - "test_score": 0.5624732538194603, - "train_loss": 0.5235076811578538, - "val_score": 0.5808439434699122 - }, - { - "epoch": 2, - "test_score": 0.5659337572123175, - "train_loss": 0.5128068581775382, - "val_score": 0.5866805240631663 - }, - { - "epoch": 3, - "test_score": 0.5697632009080036, - "train_loss": 0.5094030946493149, - "val_score": 0.5926678706027072 - }, - { - "epoch": 4, - "test_score": 0.5736462779910708, - "train_loss": 0.5063647743728426, - "val_score": 0.5980532736907034 - }, - { - "epoch": 5, - "test_score": 0.5719072191246096, - "train_loss": 0.5063130502347593, - "val_score": 0.5954887711100517 - }, - { - "epoch": 6, - "test_score": 0.5747901686703085, - "train_loss": 0.5049416814689283, - "val_score": 0.5994671346968392 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.574504443183962, - "score": 0.574504443183962 - }, - "val": { - "roc-auc": 0.6001605290639818, - "score": 0.6001605290639818 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/0", - "seed": 0, - "task_type": "binclass", - "time_seconds": 12.681254625320435, - "variant": "all_minus_cf_fisd" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1.toml deleted file mode 100644 index 39def5ff8..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1.toml +++ /dev/null @@ -1,34 +0,0 @@ -dataset = "ecom-offers" -variant = "all_minus_cf_fisd" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1/report.json deleted file mode 100644 index 3bef0e355..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1/report.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_minus_cf_fisd" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5749947559912094, - "train_loss": 0.5615878336959415, - "val_score": 0.6007862316955732 - }, - { - "epoch": 1, - "test_score": 0.5660149852626585, - "train_loss": 0.5236683112603647, - "val_score": 0.5866883184893799 - }, - { - "epoch": 2, - "test_score": 0.5685566759539047, - "train_loss": 0.5110080148334857, - "val_score": 0.5900513793834766 - }, - { - "epoch": 3, - "test_score": 0.5673698016976789, - "train_loss": 0.5102973282337189, - "val_score": 0.587963685624544 - }, - { - "epoch": 4, - "test_score": 0.5698586431697714, - "train_loss": 0.5082294896796897, - "val_score": 0.591891214074126 - }, - { - "epoch": 5, - "test_score": 0.5750190621067808, - "train_loss": 0.5046696767762855, - "val_score": 0.5987031814245977 - }, - { - "epoch": 6, - "test_score": 0.5755671523050461, - "train_loss": 0.5048941142029233, - "val_score": 0.6002077083757481 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5749947559912094, - "score": 0.5749947559912094 - }, - "val": { - "roc-auc": 0.6007862316955732, - "score": 0.6007862316955732 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/1", - "seed": 1, - "task_type": "binclass", - "time_seconds": 13.13693618774414, - "variant": "all_minus_cf_fisd" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2.toml deleted file mode 100644 index 53cab0f8d..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2.toml +++ /dev/null @@ -1,34 +0,0 @@ -dataset = "ecom-offers" -variant = "all_minus_cf_fisd" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2/report.json deleted file mode 100644 index bc6324e94..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2/report.json +++ /dev/null @@ -1,183 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 19, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_minus_cf_fisd" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.567753504819746, - "train_loss": 0.5610457912639335, - "val_score": 0.5919127657547392 - }, - { - "epoch": 1, - "test_score": 0.5677904785066432, - "train_loss": 0.5248338960938983, - "val_score": 0.5898634213098389 - }, - { - "epoch": 2, - "test_score": 0.56833937456949, - "train_loss": 0.5131386976551127, - "val_score": 0.5908690386477797 - }, - { - "epoch": 3, - "test_score": 0.5722036750078073, - "train_loss": 0.5094726416799757, - "val_score": 0.5954165538163965 - }, - { - "epoch": 4, - "test_score": 0.5688222672236289, - "train_loss": 0.5084855302616402, - "val_score": 0.5919105324581503 - }, - { - "epoch": 5, - "test_score": 0.5723461921590541, - "train_loss": 0.504546512608175, - "val_score": 0.5960148524984257 - }, - { - "epoch": 6, - "test_score": 0.5740603312130705, - "train_loss": 0.5045429435041215, - "val_score": 0.5995567245319464 - }, - { - "epoch": 7, - "test_score": 0.5703335918588237, - "train_loss": 0.504522568097821, - "val_score": 0.593099910304028 - }, - { - "epoch": 8, - "test_score": 0.5740792566328203, - "train_loss": 0.502777013513777, - "val_score": 0.5989479485195507 - }, - { - "epoch": 9, - "test_score": 0.5768869384266024, - "train_loss": 0.5016946373162446, - "val_score": 0.6014101764352952 - }, - { - "epoch": 10, - "test_score": 0.576792432207541, - "train_loss": 0.5012201556452999, - "val_score": 0.602760535900965 - }, - { - "epoch": 11, - "test_score": 0.5777785160048363, - "train_loss": 0.5014492196065409, - "val_score": 0.6048366430518524 - }, - { - "epoch": 12, - "test_score": 0.5750577993973199, - "train_loss": 0.4990316596296098, - "val_score": 0.6011327059180163 - }, - { - "epoch": 13, - "test_score": 0.5756340421647939, - "train_loss": 0.4997640174848062, - "val_score": 0.601904512581398 - }, - { - "epoch": 14, - "test_score": 0.5750410467124611, - "train_loss": 0.4982257490908658, - "val_score": 0.6003227636884607 - }, - { - "epoch": 15, - "test_score": 0.5774879212367299, - "train_loss": 0.49788987912513594, - "val_score": 0.6036308139914984 - }, - { - "epoch": 16, - "test_score": 0.5785781506332739, - "train_loss": 0.4980563951863183, - "val_score": 0.6051532294272071 - }, - { - "epoch": 17, - "test_score": 0.5771495046046669, - "train_loss": 0.49732758656696036, - "val_score": 0.6033888071921645 - }, - { - "epoch": 18, - "test_score": 0.577803860445926, - "train_loss": 0.49810293592788557, - "val_score": 0.6039634204003048 - }, - { - "epoch": 19, - "test_score": 0.5807682519060278, - "train_loss": 0.49806651362666376, - "val_score": 0.6077083676404789 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5807682519060278, - "score": 0.5807682519060278 - }, - "val": { - "roc-auc": 0.6077083676404789, - "score": 0.6077083676404789 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_cf_fisd-evaluation/2", - "seed": 2, - "task_type": "binclass", - "time_seconds": 37.24916982650757, - "variant": "all_minus_cf_fisd" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0.toml deleted file mode 100644 index 05e9af8f5..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "ecom-offers" -variant = "all_minus_esam" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0/report.json deleted file mode 100644 index 80ec1d7b3..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0/report.json +++ /dev/null @@ -1,140 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 6, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_minus_esam" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5684889616327005, - "train_loss": 0.5520850326176043, - "val_score": 0.5907757045398437 - }, - { - "epoch": 1, - "test_score": 0.570941281941853, - "train_loss": 0.5112152778440051, - "val_score": 0.594774130942852 - }, - { - "epoch": 2, - "test_score": 0.5696106662401121, - "train_loss": 0.5086778380252697, - "val_score": 0.5913595605002036 - }, - { - "epoch": 3, - "test_score": 0.569436819553518, - "train_loss": 0.5053646884582661, - "val_score": 0.5906331420889478 - }, - { - "epoch": 4, - "test_score": 0.5703592555462669, - "train_loss": 0.5050829985627422, - "val_score": 0.5940241044491734 - }, - { - "epoch": 5, - "test_score": 0.5697211688711175, - "train_loss": 0.5059763888518015, - "val_score": 0.5918137120489388 - }, - { - "epoch": 6, - "test_score": 0.573248515631535, - "train_loss": 0.5028402137535589, - "val_score": 0.5978648475829621 - }, - { - "epoch": 7, - "test_score": 0.5716950039854963, - "train_loss": 0.5033861282798979, - "val_score": 0.5947317756991297 - }, - { - "epoch": 8, - "test_score": 0.5701298289993791, - "train_loss": 0.501969211631351, - "val_score": 0.5932625687389158 - }, - { - "epoch": 9, - "test_score": 0.5698557916489444, - "train_loss": 0.501433088823601, - "val_score": 0.5918341913049523 - }, - { - "epoch": 10, - "test_score": 0.5706267467967163, - "train_loss": 0.5014634259321071, - "val_score": 0.594032311629872 - }, - { - "epoch": 11, - "test_score": 0.5717393451343564, - "train_loss": 0.4991199186554662, - "val_score": 0.5947687946170093 - }, - { - "epoch": 12, - "test_score": 0.5723682852465053, - "train_loss": 0.49935837365962843, - "val_score": 0.5952143188599449 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.573248515631535, - "score": 0.573248515631535 - }, - "val": { - "roc-auc": 0.5978648475829621, - "score": 0.5978648475829621 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/0", - "seed": 0, - "task_type": "binclass", - "time_seconds": 17.78662633895874, - "variant": "all_minus_esam" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1.toml deleted file mode 100644 index 35f230039..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "ecom-offers" -variant = "all_minus_esam" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1/report.json deleted file mode 100644 index 924afcaeb..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1/report.json +++ /dev/null @@ -1,110 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 1, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_minus_esam" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5696132883994813, - "train_loss": 0.5533099483560633, - "val_score": 0.5920076476919942 - }, - { - "epoch": 1, - "test_score": 0.5715625167488105, - "train_loss": 0.5137152550397096, - "val_score": 0.5950530313595969 - }, - { - "epoch": 2, - "test_score": 0.5694419026993401, - "train_loss": 0.5094894523973819, - "val_score": 0.5910633723040886 - }, - { - "epoch": 3, - "test_score": 0.5706710197570348, - "train_loss": 0.507937710042353, - "val_score": 0.5929211286470673 - }, - { - "epoch": 4, - "test_score": 0.5702061660716059, - "train_loss": 0.5054546721555568, - "val_score": 0.591947013321076 - }, - { - "epoch": 5, - "test_score": 0.5686125905579476, - "train_loss": 0.5028857384566907, - "val_score": 0.5881382991982672 - }, - { - "epoch": 6, - "test_score": 0.5691999232618552, - "train_loss": 0.5034620254128067, - "val_score": 0.5910225059247589 - }, - { - "epoch": 7, - "test_score": 0.571275508078092, - "train_loss": 0.5015709935515015, - "val_score": 0.5946880679474075 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5715625167488105, - "score": 0.5715625167488105 - }, - "val": { - "roc-auc": 0.5950530313595969, - "score": 0.5950530313595969 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/1", - "seed": 1, - "task_type": "binclass", - "time_seconds": 11.00665020942688, - "variant": "all_minus_esam" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2.toml deleted file mode 100644 index f5f0efcd2..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "ecom-offers" -variant = "all_minus_esam" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2/report.json deleted file mode 100644 index 350987a4c..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2/report.json +++ /dev/null @@ -1,110 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 1, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_minus_esam" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5665558350776065, - "train_loss": 0.5528011929105829, - "val_score": 0.5871859787832107 - }, - { - "epoch": 1, - "test_score": 0.5712034110933558, - "train_loss": 0.5117590449474476, - "val_score": 0.5952215125809704 - }, - { - "epoch": 2, - "test_score": 0.5690808288746189, - "train_loss": 0.5099736430026867, - "val_score": 0.5909409390049554 - }, - { - "epoch": 3, - "test_score": 0.5692677243684758, - "train_loss": 0.5080657518572278, - "val_score": 0.590635250085068 - }, - { - "epoch": 4, - "test_score": 0.570682118991906, - "train_loss": 0.5076316021106861, - "val_score": 0.5936173017363682 - }, - { - "epoch": 5, - "test_score": 0.5677198320781538, - "train_loss": 0.5045595886530699, - "val_score": 0.5892704073593417 - }, - { - "epoch": 6, - "test_score": 0.5699361952378288, - "train_loss": 0.5028332068964287, - "val_score": 0.5939281943110137 - }, - { - "epoch": 7, - "test_score": 0.5693752638974043, - "train_loss": 0.5033974564737744, - "val_score": 0.5922146514368714 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5712034110933558, - "score": 0.5712034110933558 - }, - "val": { - "roc-auc": 0.5952215125809704, - "score": 0.5952215125809704 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_esam-evaluation/2", - "seed": 2, - "task_type": "binclass", - "time_seconds": 12.06042766571045, - "variant": "all_minus_esam" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0.toml deleted file mode 100644 index 64e91ab4e..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "ecom-offers" -variant = "all_minus_mfb" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0/report.json deleted file mode 100644 index ae52fbade..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0/report.json +++ /dev/null @@ -1,140 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 6, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_minus_mfb" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5674570582349857, - "train_loss": 0.5495740638838874, - "val_score": 0.5886585062058447 - }, - { - "epoch": 1, - "test_score": 0.5725468989320472, - "train_loss": 0.5061508588216923, - "val_score": 0.5971063927901108 - }, - { - "epoch": 2, - "test_score": 0.5728618866011412, - "train_loss": 0.502218743717229, - "val_score": 0.5970249769679312 - }, - { - "epoch": 3, - "test_score": 0.5734975432846294, - "train_loss": 0.4995915039821907, - "val_score": 0.599563420736405 - }, - { - "epoch": 4, - "test_score": 0.5742185007340744, - "train_loss": 0.4979530843319716, - "val_score": 0.6018211546019538 - }, - { - "epoch": 5, - "test_score": 0.5739961750939417, - "train_loss": 0.49669913450876874, - "val_score": 0.6017292503934841 - }, - { - "epoch": 6, - "test_score": 0.5743883676894275, - "train_loss": 0.49562931612685873, - "val_score": 0.6039636304628554 - }, - { - "epoch": 7, - "test_score": 0.5742671656563625, - "train_loss": 0.4947642540490186, - "val_score": 0.603617392100118 - }, - { - "epoch": 8, - "test_score": 0.5736554896431338, - "train_loss": 0.4941260549757216, - "val_score": 0.60284559649266 - }, - { - "epoch": 9, - "test_score": 0.5724527801478807, - "train_loss": 0.4940890046181502, - "val_score": 0.600026829778591 - }, - { - "epoch": 10, - "test_score": 0.5729124050119669, - "train_loss": 0.493927875050792, - "val_score": 0.6011250257363477 - }, - { - "epoch": 11, - "test_score": 0.5726416531094415, - "train_loss": 0.49346746117980395, - "val_score": 0.6002735795691954 - }, - { - "epoch": 12, - "test_score": 0.5719451319536923, - "train_loss": 0.49327058096726734, - "val_score": 0.5997216752283494 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5743883676894275, - "score": 0.5743883676894275 - }, - "val": { - "roc-auc": 0.6039636304628554, - "score": 0.6039636304628554 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/0", - "seed": 0, - "task_type": "binclass", - "time_seconds": 23.100914239883423, - "variant": "all_minus_mfb" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1.toml deleted file mode 100644 index 71fdb2d08..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "ecom-offers" -variant = "all_minus_mfb" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1/report.json deleted file mode 100644 index f3b0be255..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1/report.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 7, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_minus_mfb" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5698811081947217, - "train_loss": 0.5510928448703554, - "val_score": 0.592789832182429 - }, - { - "epoch": 1, - "test_score": 0.5727224999238458, - "train_loss": 0.5068844297417888, - "val_score": 0.5969781735575709 - }, - { - "epoch": 2, - "test_score": 0.5736512681525181, - "train_loss": 0.5025383046379795, - "val_score": 0.5984598294878791 - }, - { - "epoch": 3, - "test_score": 0.5738685013483913, - "train_loss": 0.5001370167290723, - "val_score": 0.6001265910635086 - }, - { - "epoch": 4, - "test_score": 0.5723961495641519, - "train_loss": 0.4980512509743373, - "val_score": 0.5985286194451851 - }, - { - "epoch": 5, - "test_score": 0.5721636700302917, - "train_loss": 0.49637439515855575, - "val_score": 0.598715663562463 - }, - { - "epoch": 6, - "test_score": 0.5737822366444156, - "train_loss": 0.4955915207112277, - "val_score": 0.6027431301917421 - }, - { - "epoch": 7, - "test_score": 0.5738892213665745, - "train_loss": 0.4948982331487868, - "val_score": 0.6032786091152671 - }, - { - "epoch": 8, - "test_score": 0.5712110668068804, - "train_loss": 0.4943665370897011, - "val_score": 0.5972855982574803 - }, - { - "epoch": 9, - "test_score": 0.5718640464793927, - "train_loss": 0.4937717644152818, - "val_score": 0.5991909429814026 - }, - { - "epoch": 10, - "test_score": 0.5725859089767524, - "train_loss": 0.493560039334827, - "val_score": 0.6007264781132439 - }, - { - "epoch": 11, - "test_score": 0.5725165829263852, - "train_loss": 0.49275064689141734, - "val_score": 0.5994298578074065 - }, - { - "epoch": 12, - "test_score": 0.5718745909074944, - "train_loss": 0.49317565118824996, - "val_score": 0.598923736046632 - }, - { - "epoch": 13, - "test_score": 0.5721781352995304, - "train_loss": 0.4925886651983968, - "val_score": 0.5997260238916742 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5738892213665745, - "score": 0.5738892213665745 - }, - "val": { - "roc-auc": 0.6032786091152671, - "score": 0.6032786091152671 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/1", - "seed": 1, - "task_type": "binclass", - "time_seconds": 26.137808322906494, - "variant": "all_minus_mfb" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2.toml deleted file mode 100644 index 1224f97d7..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "ecom-offers" -variant = "all_minus_mfb" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2/report.json deleted file mode 100644 index affbd78ac..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2/report.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 3, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_minus_mfb" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5680833018002656, - "train_loss": 0.5508555427745536, - "val_score": 0.5904751455685565 - }, - { - "epoch": 1, - "test_score": 0.5738387680448112, - "train_loss": 0.5068491857360911, - "val_score": 0.6003150761361764 - }, - { - "epoch": 2, - "test_score": 0.5735754703896652, - "train_loss": 0.5023968407401332, - "val_score": 0.5990004420453122 - }, - { - "epoch": 3, - "test_score": 0.5748292097098053, - "train_loss": 0.49968371843850173, - "val_score": 0.6013164627406452 - }, - { - "epoch": 4, - "test_score": 0.5742371472006997, - "train_loss": 0.4976544071126867, - "val_score": 0.6012733298969557 - }, - { - "epoch": 5, - "test_score": 0.5717932171816329, - "train_loss": 0.495533820103716, - "val_score": 0.5970501918445993 - }, - { - "epoch": 6, - "test_score": 0.5729961312425107, - "train_loss": 0.49507507902604564, - "val_score": 0.5994206887613455 - }, - { - "epoch": 7, - "test_score": 0.5724098802568298, - "train_loss": 0.4942484376607118, - "val_score": 0.5982748491430266 - }, - { - "epoch": 8, - "test_score": 0.5712021000136711, - "train_loss": 0.493870695983922, - "val_score": 0.5960575541610907 - }, - { - "epoch": 9, - "test_score": 0.5722020694776025, - "train_loss": 0.49397959532561125, - "val_score": 0.5977455652220854 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5748292097098053, - "score": 0.5748292097098053 - }, - "val": { - "roc-auc": 0.6013164627406452, - "score": 0.6013164627406452 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_mfb-evaluation/2", - "seed": 2, - "task_type": "binclass", - "time_seconds": 19.141494274139404, - "variant": "all_minus_mfb" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0.toml deleted file mode 100644 index 81cade18c..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0.toml +++ /dev/null @@ -1,32 +0,0 @@ -dataset = "ecom-offers" -variant = "all_minus_rla" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0/report.json deleted file mode 100644 index 0faafa64d..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0/report.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_minus_rla" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5697199786711201, - "train_loss": 0.5528302910151305, - "val_score": 0.5933440803791009 - }, - { - "epoch": 1, - "test_score": 0.5666581674815466, - "train_loss": 0.5124878894399714, - "val_score": 0.5865858374472304 - }, - { - "epoch": 2, - "test_score": 0.5704257579711196, - "train_loss": 0.5099211677357003, - "val_score": 0.5922768410077242 - }, - { - "epoch": 3, - "test_score": 0.5701217269608554, - "train_loss": 0.5068588825287642, - "val_score": 0.5924421676055333 - }, - { - "epoch": 4, - "test_score": 0.5699790982283588, - "train_loss": 0.5055394018137896, - "val_score": 0.5924149442360582 - }, - { - "epoch": 5, - "test_score": 0.5687964392637903, - "train_loss": 0.506475669366342, - "val_score": 0.5901035006931401 - }, - { - "epoch": 6, - "test_score": 0.569307100151722, - "train_loss": 0.5051003059855214, - "val_score": 0.5923733334245325 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5697199786711201, - "score": 0.5697199786711201 - }, - "val": { - "roc-auc": 0.5933440803791009, - "score": 0.5933440803791009 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/0", - "seed": 0, - "task_type": "binclass", - "time_seconds": 11.543763399124146, - "variant": "all_minus_rla" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1.toml deleted file mode 100644 index 9fee2aa4d..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1.toml +++ /dev/null @@ -1,32 +0,0 @@ -dataset = "ecom-offers" -variant = "all_minus_rla" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1/report.json deleted file mode 100644 index 6f687dd89..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1/report.json +++ /dev/null @@ -1,133 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 5, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_minus_rla" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5709148433846198, - "train_loss": 0.5527302070900246, - "val_score": 0.5948060788770578 - }, - { - "epoch": 1, - "test_score": 0.569859216573416, - "train_loss": 0.5132867015070386, - "val_score": 0.5924314986391559 - }, - { - "epoch": 2, - "test_score": 0.5704153778154134, - "train_loss": 0.5095344671496639, - "val_score": 0.5928492098633522 - }, - { - "epoch": 3, - "test_score": 0.5686327836646736, - "train_loss": 0.5084150687411979, - "val_score": 0.5885501692094383 - }, - { - "epoch": 4, - "test_score": 0.5708701612930521, - "train_loss": 0.5059906867919145, - "val_score": 0.5926585762561773 - }, - { - "epoch": 5, - "test_score": 0.573474359180514, - "train_loss": 0.5031700454376362, - "val_score": 0.5984686926533844 - }, - { - "epoch": 6, - "test_score": 0.5729537334670838, - "train_loss": 0.5041169088195872, - "val_score": 0.5971772428345342 - }, - { - "epoch": 7, - "test_score": 0.5720949793731522, - "train_loss": 0.5038585844967101, - "val_score": 0.5956662702798624 - }, - { - "epoch": 8, - "test_score": 0.5696481420426334, - "train_loss": 0.5019371962105786, - "val_score": 0.591666911493955 - }, - { - "epoch": 9, - "test_score": 0.5705172298000838, - "train_loss": 0.503837333785163, - "val_score": 0.5938676336462538 - }, - { - "epoch": 10, - "test_score": 0.5720236944519559, - "train_loss": 0.5018296633605603, - "val_score": 0.5940563214108562 - }, - { - "epoch": 11, - "test_score": 0.5724766182420986, - "train_loss": 0.5006160101404896, - "val_score": 0.5938442135145341 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.573474359180514, - "score": 0.573474359180514 - }, - "val": { - "roc-auc": 0.5984686926533844, - "score": 0.5984686926533844 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/1", - "seed": 1, - "task_type": "binclass", - "time_seconds": 19.83189105987549, - "variant": "all_minus_rla" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2.toml deleted file mode 100644 index 2be994558..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2.toml +++ /dev/null @@ -1,32 +0,0 @@ -dataset = "ecom-offers" -variant = "all_minus_rla" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2/report.json deleted file mode 100644 index 0b806f5b4..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2/report.json +++ /dev/null @@ -1,109 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 1, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_minus_rla" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5640513629321112, - "train_loss": 0.5529496222734451, - "val_score": 0.5826846405226108 - }, - { - "epoch": 1, - "test_score": 0.5729159973083131, - "train_loss": 0.5133549846984722, - "val_score": 0.5991167798451749 - }, - { - "epoch": 2, - "test_score": 0.5704932894230534, - "train_loss": 0.5109604150056839, - "val_score": 0.594155806297681 - }, - { - "epoch": 3, - "test_score": 0.571275858319237, - "train_loss": 0.5076655331585143, - "val_score": 0.5947095790896337 - }, - { - "epoch": 4, - "test_score": 0.5672518014265862, - "train_loss": 0.5066016233629651, - "val_score": 0.5872497604070774 - }, - { - "epoch": 5, - "test_score": 0.5696887049263977, - "train_loss": 0.5043161257549569, - "val_score": 0.5932295852331895 - }, - { - "epoch": 6, - "test_score": 0.5696682948561305, - "train_loss": 0.5044200701846017, - "val_score": 0.5937620274630029 - }, - { - "epoch": 7, - "test_score": 0.5669393181366531, - "train_loss": 0.5049267328447766, - "val_score": 0.5873475905906542 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5729159973083131, - "score": 0.5729159973083131 - }, - "val": { - "roc-auc": 0.5991167798451749, - "score": 0.5991167798451749 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/all_minus_rla-evaluation/2", - "seed": 2, - "task_type": "binclass", - "time_seconds": 13.887128829956055, - "variant": "all_minus_rla" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0.toml deleted file mode 100644 index 6624bf9a2..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0.toml +++ /dev/null @@ -1,27 +0,0 @@ -dataset = "ecom-offers" -variant = "baseline_plr" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0/report.json deleted file mode 100644 index 41fd7246b..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0/report.json +++ /dev/null @@ -1,176 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 14, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "baseline_plr" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5750006171063008, - "train_loss": 0.5606814733258, - "val_score": 0.601598805234971 - }, - { - "epoch": 1, - "test_score": 0.5678999118173383, - "train_loss": 0.5204960731444536, - "val_score": 0.5896181788101665 - }, - { - "epoch": 2, - "test_score": 0.5716925058052935, - "train_loss": 0.506363250039242, - "val_score": 0.5955156628018154 - }, - { - "epoch": 3, - "test_score": 0.5729560239821828, - "train_loss": 0.501323691672749, - "val_score": 0.5973503675438647 - }, - { - "epoch": 4, - "test_score": 0.5743658530728106, - "train_loss": 0.49991664235238675, - "val_score": 0.5996836539067204 - }, - { - "epoch": 5, - "test_score": 0.5756228344481519, - "train_loss": 0.49864044068036256, - "val_score": 0.6016151643167498 - }, - { - "epoch": 6, - "test_score": 0.5758462325080738, - "train_loss": 0.4971861519195415, - "val_score": 0.602125303063138 - }, - { - "epoch": 7, - "test_score": 0.5772494969018412, - "train_loss": 0.4960625370343526, - "val_score": 0.6039556112328598 - }, - { - "epoch": 8, - "test_score": 0.5749955897511035, - "train_loss": 0.4952058858341641, - "val_score": 0.6014264618109157 - }, - { - "epoch": 9, - "test_score": 0.5751647407267706, - "train_loss": 0.4940291941165924, - "val_score": 0.6030069761257056 - }, - { - "epoch": 10, - "test_score": 0.5740187919933709, - "train_loss": 0.4936152751798983, - "val_score": 0.602776920779899 - }, - { - "epoch": 11, - "test_score": 0.5753113119967586, - "train_loss": 0.49395177761713666, - "val_score": 0.6049119965425326 - }, - { - "epoch": 12, - "test_score": 0.5730560534731072, - "train_loss": 0.4931856112347709, - "val_score": 0.602283498590156 - }, - { - "epoch": 13, - "test_score": 0.5752409321234768, - "train_loss": 0.4932894745358714, - "val_score": 0.6063850878172757 - }, - { - "epoch": 14, - "test_score": 0.5767959687132622, - "train_loss": 0.49289950397279525, - "val_score": 0.6083677687275297 - }, - { - "epoch": 15, - "test_score": 0.5761139159262311, - "train_loss": 0.49263357288307613, - "val_score": 0.607897663480885 - }, - { - "epoch": 16, - "test_score": 0.5763534684715339, - "train_loss": 0.4920575221379598, - "val_score": 0.6080092030098587 - }, - { - "epoch": 17, - "test_score": 0.5754996890416538, - "train_loss": 0.49198202733640317, - "val_score": 0.6067648698525406 - }, - { - "epoch": 18, - "test_score": 0.5760725502773644, - "train_loss": 0.49181015182424476, - "val_score": 0.6074467255206407 - }, - { - "epoch": 19, - "test_score": 0.5760798929434939, - "train_loss": 0.4918224579758114, - "val_score": 0.6080325310088809 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5767959687132622, - "score": 0.5767959687132622 - }, - "val": { - "roc-auc": 0.6083677687275297, - "score": 0.6083677687275297 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/0", - "seed": 0, - "task_type": "binclass", - "time_seconds": 25.43160104751587, - "variant": "baseline_plr" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1.toml deleted file mode 100644 index 71cc0e428..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1.toml +++ /dev/null @@ -1,27 +0,0 @@ -dataset = "ecom-offers" -variant = "baseline_plr" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1/report.json deleted file mode 100644 index 8223ba01e..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1/report.json +++ /dev/null @@ -1,176 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 14, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "baseline_plr" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5746579324914347, - "train_loss": 0.5609680237593474, - "val_score": 0.6008386257180214 - }, - { - "epoch": 1, - "test_score": 0.5635432591144864, - "train_loss": 0.5211676916590443, - "val_score": 0.5821124669883017 - }, - { - "epoch": 2, - "test_score": 0.5713524464492756, - "train_loss": 0.5065342673548946, - "val_score": 0.5941694087691489 - }, - { - "epoch": 3, - "test_score": 0.5728148984970787, - "train_loss": 0.5021586578201365, - "val_score": 0.5963798159106152 - }, - { - "epoch": 4, - "test_score": 0.5745534273526035, - "train_loss": 0.49993850345964785, - "val_score": 0.5985960937475701 - }, - { - "epoch": 5, - "test_score": 0.5757297199869778, - "train_loss": 0.498760970654311, - "val_score": 0.6008029003432156 - }, - { - "epoch": 6, - "test_score": 0.5770634072725658, - "train_loss": 0.4972201293265378, - "val_score": 0.6031019059719633 - }, - { - "epoch": 7, - "test_score": 0.5774046103363937, - "train_loss": 0.49641228274062826, - "val_score": 0.603039885925274 - }, - { - "epoch": 8, - "test_score": 0.5755726662784714, - "train_loss": 0.4953731486090907, - "val_score": 0.6003639285777305 - }, - { - "epoch": 9, - "test_score": 0.5757936467446486, - "train_loss": 0.4951117623735357, - "val_score": 0.6030508239191282 - }, - { - "epoch": 10, - "test_score": 0.5747242675444127, - "train_loss": 0.49429288340939415, - "val_score": 0.6015486150266484 - }, - { - "epoch": 11, - "test_score": 0.574505664378751, - "train_loss": 0.4939393456335421, - "val_score": 0.6033995977737026 - }, - { - "epoch": 12, - "test_score": 0.5752168491704052, - "train_loss": 0.4929683832106767, - "val_score": 0.6050918211416296 - }, - { - "epoch": 13, - "test_score": 0.5763590940262089, - "train_loss": 0.49330422392597906, - "val_score": 0.6073785694362939 - }, - { - "epoch": 14, - "test_score": 0.5763749354641947, - "train_loss": 0.4932381847390422, - "val_score": 0.6080396657649799 - }, - { - "epoch": 15, - "test_score": 0.5754465980631256, - "train_loss": 0.4925268402806035, - "val_score": 0.6063322405019554 - }, - { - "epoch": 16, - "test_score": 0.5743056611875272, - "train_loss": 0.4924654452889054, - "val_score": 0.6050064472987629 - }, - { - "epoch": 17, - "test_score": 0.5731771749197139, - "train_loss": 0.4926512842928922, - "val_score": 0.6022254992143882 - }, - { - "epoch": 18, - "test_score": 0.575151363374717, - "train_loss": 0.49213752647240955, - "val_score": 0.6055324254985157 - }, - { - "epoch": 19, - "test_score": 0.5757693530269941, - "train_loss": 0.49162033752158835, - "val_score": 0.6063474645089002 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5763749354641947, - "score": 0.5763749354641947 - }, - "val": { - "roc-auc": 0.6080396657649799, - "score": 0.6080396657649799 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/1", - "seed": 1, - "task_type": "binclass", - "time_seconds": 25.662135124206543, - "variant": "baseline_plr" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2.toml deleted file mode 100644 index e657e755b..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2.toml +++ /dev/null @@ -1,27 +0,0 @@ -dataset = "ecom-offers" -variant = "baseline_plr" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2/report.json deleted file mode 100644 index b088b8366..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2/report.json +++ /dev/null @@ -1,176 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 17, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "baseline_plr" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5647947637101525, - "train_loss": 0.5599922913092154, - "val_score": 0.5875559947525343 - }, - { - "epoch": 1, - "test_score": 0.5665267030729836, - "train_loss": 0.5211185868139621, - "val_score": 0.5884676146271168 - }, - { - "epoch": 2, - "test_score": 0.57200711533793, - "train_loss": 0.5062609061046883, - "val_score": 0.5959624216228985 - }, - { - "epoch": 3, - "test_score": 0.5730880060037655, - "train_loss": 0.5022778091607271, - "val_score": 0.5964357588845735 - }, - { - "epoch": 4, - "test_score": 0.5750484017765074, - "train_loss": 0.4995520931703073, - "val_score": 0.5986645593977806 - }, - { - "epoch": 5, - "test_score": 0.5769945833378225, - "train_loss": 0.4983021484480964, - "val_score": 0.6013675447934803 - }, - { - "epoch": 6, - "test_score": 0.5764173270406633, - "train_loss": 0.49657544272917287, - "val_score": 0.6021789390343494 - }, - { - "epoch": 7, - "test_score": 0.5755138350645392, - "train_loss": 0.4954980076463134, - "val_score": 0.6006230167791922 - }, - { - "epoch": 8, - "test_score": 0.5753642913940369, - "train_loss": 0.4957014432659856, - "val_score": 0.6007992371471607 - }, - { - "epoch": 9, - "test_score": 0.5751326177247584, - "train_loss": 0.49462021776923426, - "val_score": 0.6006985619058831 - }, - { - "epoch": 10, - "test_score": 0.574050970786008, - "train_loss": 0.4937642288428766, - "val_score": 0.5991349557837495 - }, - { - "epoch": 11, - "test_score": 0.5759820578838146, - "train_loss": 0.4934336228503121, - "val_score": 0.6043515349167372 - }, - { - "epoch": 12, - "test_score": 0.5740183425688927, - "train_loss": 0.4933987199156373, - "val_score": 0.6007254167445679 - }, - { - "epoch": 13, - "test_score": 0.5732814321002122, - "train_loss": 0.49324297849778775, - "val_score": 0.6003006149879674 - }, - { - "epoch": 14, - "test_score": 0.5745096038167632, - "train_loss": 0.49280598373324785, - "val_score": 0.6019991365470997 - }, - { - "epoch": 15, - "test_score": 0.5742709966126041, - "train_loss": 0.4923618893932413, - "val_score": 0.6030152385860228 - }, - { - "epoch": 16, - "test_score": 0.5750240956609363, - "train_loss": 0.49230552713076275, - "val_score": 0.6044452412407713 - }, - { - "epoch": 17, - "test_score": 0.57621293188747, - "train_loss": 0.49252393620985524, - "val_score": 0.6069789751857458 - }, - { - "epoch": 18, - "test_score": 0.574770899208372, - "train_loss": 0.4919488087848381, - "val_score": 0.6040690523807111 - }, - { - "epoch": 19, - "test_score": 0.5760533954961569, - "train_loss": 0.49158883149977084, - "val_score": 0.606631240588 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.57621293188747, - "score": 0.57621293188747 - }, - "val": { - "roc-auc": 0.6069789751857458, - "score": 0.6069789751857458 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/baseline_plr-evaluation/2", - "seed": 2, - "task_type": "binclass", - "time_seconds": 28.923240184783936, - "variant": "baseline_plr" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0.toml deleted file mode 100644 index ee951091f..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0.toml +++ /dev/null @@ -1,28 +0,0 @@ -dataset = "ecom-offers" -variant = "cf_fisd_only" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0/report.json deleted file mode 100644 index ccfa17323..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0/report.json +++ /dev/null @@ -1,141 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 7, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "cf_fisd_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5694692246081338, - "train_loss": 0.5502510170141856, - "val_score": 0.5929137985696494 - }, - { - "epoch": 1, - "test_score": 0.5730968550167668, - "train_loss": 0.5064160205699779, - "val_score": 0.5983763572638899 - }, - { - "epoch": 2, - "test_score": 0.5747250517126401, - "train_loss": 0.5026354524824355, - "val_score": 0.6007589641024034 - }, - { - "epoch": 3, - "test_score": 0.5731190627849467, - "train_loss": 0.4991984836481236, - "val_score": 0.5989815069333099 - }, - { - "epoch": 4, - "test_score": 0.57317102865254, - "train_loss": 0.49786143722357573, - "val_score": 0.5998851849693626 - }, - { - "epoch": 5, - "test_score": 0.5740694126870087, - "train_loss": 0.49681372388645456, - "val_score": 0.6021412825582028 - }, - { - "epoch": 6, - "test_score": 0.5729867460196147, - "train_loss": 0.4954415603920265, - "val_score": 0.6008536986273423 - }, - { - "epoch": 7, - "test_score": 0.5731309058948163, - "train_loss": 0.494728273263684, - "val_score": 0.6021670244339009 - }, - { - "epoch": 8, - "test_score": 0.5719955697780599, - "train_loss": 0.494201863805453, - "val_score": 0.599877924912795 - }, - { - "epoch": 9, - "test_score": 0.5722517107358259, - "train_loss": 0.49334189185389765, - "val_score": 0.6005756126636438 - }, - { - "epoch": 10, - "test_score": 0.5715275670218046, - "train_loss": 0.4932152231534322, - "val_score": 0.5983085181160228 - }, - { - "epoch": 11, - "test_score": 0.5720281948956959, - "train_loss": 0.4935942192872365, - "val_score": 0.6002474101977806 - }, - { - "epoch": 12, - "test_score": 0.5695525943985742, - "train_loss": 0.49303652346134186, - "val_score": 0.594618124488728 - }, - { - "epoch": 13, - "test_score": 0.5718520545945235, - "train_loss": 0.49319977561632794, - "val_score": 0.5995587993602954 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5731309058948163, - "score": 0.5731309058948163 - }, - "val": { - "roc-auc": 0.6021670244339009, - "score": 0.6021670244339009 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/0", - "seed": 0, - "task_type": "binclass", - "time_seconds": 18.104774713516235, - "variant": "cf_fisd_only" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1.toml deleted file mode 100644 index 1fad05de8..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1.toml +++ /dev/null @@ -1,28 +0,0 @@ -dataset = "ecom-offers" -variant = "cf_fisd_only" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1/report.json deleted file mode 100644 index 51f0f9295..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1/report.json +++ /dev/null @@ -1,135 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 6, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "cf_fisd_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5707511164974824, - "train_loss": 0.5509031525364628, - "val_score": 0.5948029721624961 - }, - { - "epoch": 1, - "test_score": 0.5710755947717638, - "train_loss": 0.5070444698686953, - "val_score": 0.5939918985434145 - }, - { - "epoch": 2, - "test_score": 0.5741403597649766, - "train_loss": 0.502915170457628, - "val_score": 0.5995196540197563 - }, - { - "epoch": 3, - "test_score": 0.5740006538413278, - "train_loss": 0.500177988299617, - "val_score": 0.6005159475287041 - }, - { - "epoch": 4, - "test_score": 0.5736018221614818, - "train_loss": 0.49774023283410956, - "val_score": 0.6005149893486494 - }, - { - "epoch": 5, - "test_score": 0.5729031065744875, - "train_loss": 0.4964937291763447, - "val_score": 0.5996165481351259 - }, - { - "epoch": 6, - "test_score": 0.5730458902809421, - "train_loss": 0.4950112937777131, - "val_score": 0.6005870260622175 - }, - { - "epoch": 7, - "test_score": 0.5718939161600558, - "train_loss": 0.49456944896115196, - "val_score": 0.5993327167764075 - }, - { - "epoch": 8, - "test_score": 0.5709125652674374, - "train_loss": 0.4939544118112988, - "val_score": 0.5975769476443196 - }, - { - "epoch": 9, - "test_score": 0.5721842846661835, - "train_loss": 0.49402084008411123, - "val_score": 0.6001584616062485 - }, - { - "epoch": 10, - "test_score": 0.571063903536373, - "train_loss": 0.493623247301137, - "val_score": 0.5976913875106123 - }, - { - "epoch": 11, - "test_score": 0.5712434129713921, - "train_loss": 0.49344748148211726, - "val_score": 0.5978700070140254 - }, - { - "epoch": 12, - "test_score": 0.5726795101478993, - "train_loss": 0.49258249280629335, - "val_score": 0.6005388001230066 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5730458902809421, - "score": 0.5730458902809421 - }, - "val": { - "roc-auc": 0.6005870260622175, - "score": 0.6005870260622175 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/1", - "seed": 1, - "task_type": "binclass", - "time_seconds": 16.94504404067993, - "variant": "cf_fisd_only" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2.toml deleted file mode 100644 index 42bdd942d..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2.toml +++ /dev/null @@ -1,28 +0,0 @@ -dataset = "ecom-offers" -variant = "cf_fisd_only" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2/report.json deleted file mode 100644 index 75d607f7d..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2/report.json +++ /dev/null @@ -1,135 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 6, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "cf_fisd_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5664456392950381, - "train_loss": 0.5500196737271769, - "val_score": 0.5873379829929527 - }, - { - "epoch": 1, - "test_score": 0.5735965127536811, - "train_loss": 0.5067310901703658, - "val_score": 0.599813325150574 - }, - { - "epoch": 2, - "test_score": 0.5743384784728712, - "train_loss": 0.5025817271735933, - "val_score": 0.60041159435014 - }, - { - "epoch": 3, - "test_score": 0.5735590834433473, - "train_loss": 0.4996792955531014, - "val_score": 0.598594505379864 - }, - { - "epoch": 4, - "test_score": 0.5730344687002382, - "train_loss": 0.4973104816895944, - "val_score": 0.598631505871204 - }, - { - "epoch": 5, - "test_score": 0.5734397875899657, - "train_loss": 0.49601822153285696, - "val_score": 0.5994892354883299 - }, - { - "epoch": 6, - "test_score": 0.5737641046913308, - "train_loss": 0.49453101831453816, - "val_score": 0.6010747507659433 - }, - { - "epoch": 7, - "test_score": 0.5717324239973925, - "train_loss": 0.4938915134580047, - "val_score": 0.5972482918855844 - }, - { - "epoch": 8, - "test_score": 0.5721690042339257, - "train_loss": 0.49456112141962405, - "val_score": 0.5987011839877145 - }, - { - "epoch": 9, - "test_score": 0.5718806255934186, - "train_loss": 0.4939179475660677, - "val_score": 0.5980858628684842 - }, - { - "epoch": 10, - "test_score": 0.5710394114520523, - "train_loss": 0.4932858695586522, - "val_score": 0.5962455085494205 - }, - { - "epoch": 11, - "test_score": 0.5725301958388551, - "train_loss": 0.4931839207808177, - "val_score": 0.5989351936689005 - }, - { - "epoch": 12, - "test_score": 0.5714935905312548, - "train_loss": 0.4932064943843418, - "val_score": 0.5970939990996351 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5737641046913308, - "score": 0.5737641046913308 - }, - "val": { - "roc-auc": 0.6010747507659433, - "score": 0.6010747507659433 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/cf_fisd_only-evaluation/2", - "seed": 2, - "task_type": "binclass", - "time_seconds": 18.815613985061646, - "variant": "cf_fisd_only" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0.toml deleted file mode 100644 index 65294a334..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "ecom-offers" -variant = "esam_only" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0/report.json deleted file mode 100644 index 09447b34f..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0/report.json +++ /dev/null @@ -1,178 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 14, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "esam_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5751306650528877, - "train_loss": 0.56087240907881, - "val_score": 0.6017959286693619 - }, - { - "epoch": 1, - "test_score": 0.5677375920937394, - "train_loss": 0.5205924102553615, - "val_score": 0.5893268183674035 - }, - { - "epoch": 2, - "test_score": 0.5716916720453995, - "train_loss": 0.5064562239028789, - "val_score": 0.5955753389926788 - }, - { - "epoch": 3, - "test_score": 0.5727886149138035, - "train_loss": 0.501467732919587, - "val_score": 0.5971922457230048 - }, - { - "epoch": 4, - "test_score": 0.574375461458206, - "train_loss": 0.49999182643713774, - "val_score": 0.5995666085277406 - }, - { - "epoch": 5, - "test_score": 0.5756433963928981, - "train_loss": 0.49881415676187585, - "val_score": 0.6016288699768387 - }, - { - "epoch": 6, - "test_score": 0.5759478210370156, - "train_loss": 0.4973221161851176, - "val_score": 0.6020441488978194 - }, - { - "epoch": 7, - "test_score": 0.577360008831284, - "train_loss": 0.4961576544576221, - "val_score": 0.6040952438639735 - }, - { - "epoch": 8, - "test_score": 0.575118840539893, - "train_loss": 0.4953662283994533, - "val_score": 0.6016707719276889 - }, - { - "epoch": 9, - "test_score": 0.5755175668374476, - "train_loss": 0.4941921719798335, - "val_score": 0.603020066339375 - }, - { - "epoch": 10, - "test_score": 0.5742576099621128, - "train_loss": 0.49376690994810174, - "val_score": 0.6027129143522503 - }, - { - "epoch": 11, - "test_score": 0.5748242226478373, - "train_loss": 0.4941109347122687, - "val_score": 0.6034782422443395 - }, - { - "epoch": 12, - "test_score": 0.5729161336853962, - "train_loss": 0.49326626901273374, - "val_score": 0.6008168418630104 - }, - { - "epoch": 13, - "test_score": 0.5751381038028711, - "train_loss": 0.49349481971175585, - "val_score": 0.6060124773911888 - }, - { - "epoch": 14, - "test_score": 0.5766903074687046, - "train_loss": 0.4929675846188157, - "val_score": 0.6082039420500369 - }, - { - "epoch": 15, - "test_score": 0.5762027593968676, - "train_loss": 0.4927098403374354, - "val_score": 0.6080009331789258 - }, - { - "epoch": 16, - "test_score": 0.5763376766252147, - "train_loss": 0.4920907649728987, - "val_score": 0.6077780973513 - }, - { - "epoch": 17, - "test_score": 0.5756898017948786, - "train_loss": 0.49206242130862343, - "val_score": 0.6070591195766991 - }, - { - "epoch": 18, - "test_score": 0.5761874479698181, - "train_loss": 0.4918863221451088, - "val_score": 0.6075633802569831 - }, - { - "epoch": 19, - "test_score": 0.575851625601812, - "train_loss": 0.49199968190104876, - "val_score": 0.6074322643724318 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5766903074687046, - "score": 0.5766903074687046 - }, - "val": { - "roc-auc": 0.6082039420500369, - "score": 0.6082039420500369 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/0", - "seed": 0, - "task_type": "binclass", - "time_seconds": 33.07249736785889, - "variant": "esam_only" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1.toml deleted file mode 100644 index e2261c903..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "ecom-offers" -variant = "esam_only" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1/report.json deleted file mode 100644 index f2ab418cc..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1/report.json +++ /dev/null @@ -1,178 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 14, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "esam_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5746909419444866, - "train_loss": 0.5611117923701251, - "val_score": 0.6008023401764144 - }, - { - "epoch": 1, - "test_score": 0.5635703609602598, - "train_loss": 0.5212651198660886, - "val_score": 0.58227185655508 - }, - { - "epoch": 2, - "test_score": 0.5711950301017077, - "train_loss": 0.5066843419163315, - "val_score": 0.5939066757981717 - }, - { - "epoch": 3, - "test_score": 0.5726653734234515, - "train_loss": 0.5021584519633541, - "val_score": 0.5962150089412203 - }, - { - "epoch": 4, - "test_score": 0.5744955569772109, - "train_loss": 0.4999331335226695, - "val_score": 0.5984644729758362 - }, - { - "epoch": 5, - "test_score": 0.5756790187068817, - "train_loss": 0.4988091301034998, - "val_score": 0.6007560158560814 - }, - { - "epoch": 6, - "test_score": 0.5770970552183247, - "train_loss": 0.4973242293905329, - "val_score": 0.6032151444278802 - }, - { - "epoch": 7, - "test_score": 0.5772226554123173, - "train_loss": 0.4965477806550485, - "val_score": 0.6029452029946458 - }, - { - "epoch": 8, - "test_score": 0.5755433328077031, - "train_loss": 0.49546305117783723, - "val_score": 0.6004737360119901 - }, - { - "epoch": 9, - "test_score": 0.5763837038907378, - "train_loss": 0.4952508856852849, - "val_score": 0.6034021959157737 - }, - { - "epoch": 10, - "test_score": 0.5750985296529589, - "train_loss": 0.4944774690601561, - "val_score": 0.6018824081045995 - }, - { - "epoch": 11, - "test_score": 0.5751528976169009, - "train_loss": 0.4940486892505928, - "val_score": 0.6037345369824112 - }, - { - "epoch": 12, - "test_score": 0.5758723146252036, - "train_loss": 0.49305023142585047, - "val_score": 0.6053632403833327 - }, - { - "epoch": 13, - "test_score": 0.5768479128845014, - "train_loss": 0.493441888027721, - "val_score": 0.6077879002703204 - }, - { - "epoch": 14, - "test_score": 0.5767053647384629, - "train_loss": 0.4933244977836256, - "val_score": 0.6084306806187314 - }, - { - "epoch": 15, - "test_score": 0.5760965774398111, - "train_loss": 0.4926842865016725, - "val_score": 0.607144147000623 - }, - { - "epoch": 16, - "test_score": 0.5745206410620511, - "train_loss": 0.49252080696600453, - "val_score": 0.6051915308322366 - }, - { - "epoch": 17, - "test_score": 0.5735734092360238, - "train_loss": 0.4926967184852671, - "val_score": 0.6027158662838801 - }, - { - "epoch": 18, - "test_score": 0.5757706083160539, - "train_loss": 0.49223133700865285, - "val_score": 0.6061065006517025 - }, - { - "epoch": 19, - "test_score": 0.5757061701443216, - "train_loss": 0.49164940140865465, - "val_score": 0.6061247797788983 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5767053647384629, - "score": 0.5767053647384629 - }, - "val": { - "roc-auc": 0.6084306806187314, - "score": 0.6084306806187314 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/1", - "seed": 1, - "task_type": "binclass", - "time_seconds": 32.38373279571533, - "variant": "esam_only" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2.toml deleted file mode 100644 index 5b83af7db..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "ecom-offers" -variant = "esam_only" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2/report.json deleted file mode 100644 index 00e25e38e..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2/report.json +++ /dev/null @@ -1,178 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 19, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "esam_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5646824633812344, - "train_loss": 0.5600938432746463, - "val_score": 0.5874748921815265 - }, - { - "epoch": 1, - "test_score": 0.5663432046082858, - "train_loss": 0.5211920037313744, - "val_score": 0.5882059172276601 - }, - { - "epoch": 2, - "test_score": 0.5718646601762665, - "train_loss": 0.5063478036059273, - "val_score": 0.5957459724338674 - }, - { - "epoch": 3, - "test_score": 0.5730667714720417, - "train_loss": 0.5023864270360382, - "val_score": 0.5964439034150377 - }, - { - "epoch": 4, - "test_score": 0.5750929908837003, - "train_loss": 0.49960060472841616, - "val_score": 0.5986022408411512 - }, - { - "epoch": 5, - "test_score": 0.5769539925587457, - "train_loss": 0.49839139646954006, - "val_score": 0.6012458485559276 - }, - { - "epoch": 6, - "test_score": 0.5764294336062614, - "train_loss": 0.49663131766849095, - "val_score": 0.6021241458764567 - }, - { - "epoch": 7, - "test_score": 0.5755681100441066, - "train_loss": 0.4956015679571364, - "val_score": 0.6006771318404308 - }, - { - "epoch": 8, - "test_score": 0.5754694102297417, - "train_loss": 0.49574901163578033, - "val_score": 0.6008344318376285 - }, - { - "epoch": 9, - "test_score": 0.5752777756322495, - "train_loss": 0.49462479960035394, - "val_score": 0.6007017275853714 - }, - { - "epoch": 10, - "test_score": 0.5740121312126565, - "train_loss": 0.49387454986572266, - "val_score": 0.5990072156412368 - }, - { - "epoch": 11, - "test_score": 0.5758652261163651, - "train_loss": 0.4934555744683301, - "val_score": 0.6041306854700706 - }, - { - "epoch": 12, - "test_score": 0.5736422331707673, - "train_loss": 0.49341002437803483, - "val_score": 0.600174452157237 - }, - { - "epoch": 13, - "test_score": 0.5734670599070928, - "train_loss": 0.4933059408708855, - "val_score": 0.6011017567022521 - }, - { - "epoch": 14, - "test_score": 0.5745859718837815, - "train_loss": 0.4929176900121901, - "val_score": 0.6021131857707549 - }, - { - "epoch": 15, - "test_score": 0.5744547430356346, - "train_loss": 0.49251379293424113, - "val_score": 0.6033664152613495 - }, - { - "epoch": 16, - "test_score": 0.5753802196174392, - "train_loss": 0.4923371247671269, - "val_score": 0.6050771020218675 - }, - { - "epoch": 17, - "test_score": 0.5764116147005718, - "train_loss": 0.4924943822401541, - "val_score": 0.607187338809239 - }, - { - "epoch": 18, - "test_score": 0.5751273269138326, - "train_loss": 0.49204753670427537, - "val_score": 0.6045122806768237 - }, - { - "epoch": 19, - "test_score": 0.5764039992802762, - "train_loss": 0.4917129118133474, - "val_score": 0.6073509185711023 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5764039992802762, - "score": 0.5764039992802762 - }, - "val": { - "roc-auc": 0.6073509185711023, - "score": 0.6073509185711023 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/esam_only-evaluation/2", - "seed": 2, - "task_type": "binclass", - "time_seconds": 36.74677538871765, - "variant": "esam_only" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0.toml deleted file mode 100644 index ed6cb5c3a..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "ecom-offers" -variant = "mfb_only" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0/report.json deleted file mode 100644 index 163bc3268..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0/report.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "mfb_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5756126061669247, - "train_loss": 0.5612001937848551, - "val_score": 0.6017022260306357 - }, - { - "epoch": 1, - "test_score": 0.5678979560459885, - "train_loss": 0.5231215937270058, - "val_score": 0.5896429772470416 - }, - { - "epoch": 2, - "test_score": 0.5704295176393404, - "train_loss": 0.5125225379511162, - "val_score": 0.5927430766810713 - }, - { - "epoch": 3, - "test_score": 0.5706981309012455, - "train_loss": 0.508107524227213, - "val_score": 0.5932826057579811 - }, - { - "epoch": 4, - "test_score": 0.5717370918130072, - "train_loss": 0.5076041861816689, - "val_score": 0.5947484922557751 - }, - { - "epoch": 5, - "test_score": 0.5731379541104258, - "train_loss": 0.5074608066567668, - "val_score": 0.5966676052900207 - }, - { - "epoch": 6, - "test_score": 0.5749101960007718, - "train_loss": 0.5042634120693913, - "val_score": 0.5994873007016811 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5756126061669247, - "score": 0.5756126061669247 - }, - "val": { - "roc-auc": 0.6017022260306357, - "score": 0.6017022260306357 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/0", - "seed": 0, - "task_type": "binclass", - "time_seconds": 9.245736360549927, - "variant": "mfb_only" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1.toml deleted file mode 100644 index dc012a660..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "ecom-offers" -variant = "mfb_only" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1/report.json deleted file mode 100644 index d424359d0..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1/report.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "mfb_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5758805406428937, - "train_loss": 0.5612830320994059, - "val_score": 0.6019166519856285 - }, - { - "epoch": 1, - "test_score": 0.5639916173718269, - "train_loss": 0.5240427851676941, - "val_score": 0.5835630336611454 - }, - { - "epoch": 2, - "test_score": 0.5662793181438439, - "train_loss": 0.5127950227922864, - "val_score": 0.5855571279699732 - }, - { - "epoch": 3, - "test_score": 0.5687952242679598, - "train_loss": 0.5094026978369113, - "val_score": 0.5900073252128114 - }, - { - "epoch": 4, - "test_score": 0.5723007940878004, - "train_loss": 0.5075066724309215, - "val_score": 0.5948114889090584 - }, - { - "epoch": 5, - "test_score": 0.5723376406960523, - "train_loss": 0.5056416751058014, - "val_score": 0.5956702614683207 - }, - { - "epoch": 6, - "test_score": 0.5747176408579691, - "train_loss": 0.5049899849626753, - "val_score": 0.5981938313341004 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5758805406428937, - "score": 0.5758805406428937 - }, - "val": { - "roc-auc": 0.6019166519856285, - "score": 0.6019166519856285 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/1", - "seed": 1, - "task_type": "binclass", - "time_seconds": 9.183435678482056, - "variant": "mfb_only" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2.toml deleted file mode 100644 index 976fb7e3c..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "ecom-offers" -variant = "mfb_only" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2/report.json deleted file mode 100644 index ae147922f..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2/report.json +++ /dev/null @@ -1,178 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 17, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "mfb_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5655640513381363, - "train_loss": 0.5601579028147238, - "val_score": 0.5890461121497865 - }, - { - "epoch": 1, - "test_score": 0.566298851061509, - "train_loss": 0.5232984291182624, - "val_score": 0.5876600125680793 - }, - { - "epoch": 2, - "test_score": 0.565342370389669, - "train_loss": 0.5126102323885318, - "val_score": 0.5864489282586565 - }, - { - "epoch": 3, - "test_score": 0.56817462485423, - "train_loss": 0.5099104096492132, - "val_score": 0.5899897942031197 - }, - { - "epoch": 4, - "test_score": 0.5717861255733153, - "train_loss": 0.5082687989429191, - "val_score": 0.5945968860592866 - }, - { - "epoch": 5, - "test_score": 0.5718616877757521, - "train_loss": 0.5065136921626551, - "val_score": 0.5957169395782125 - }, - { - "epoch": 6, - "test_score": 0.5747729355661799, - "train_loss": 0.5042106216704404, - "val_score": 0.6006416017869438 - }, - { - "epoch": 7, - "test_score": 0.5739895267111441, - "train_loss": 0.5042047484053506, - "val_score": 0.5990293459151907 - }, - { - "epoch": 8, - "test_score": 0.5723069682502868, - "train_loss": 0.5039668894476361, - "val_score": 0.5963988800083944 - }, - { - "epoch": 9, - "test_score": 0.5764932115889327, - "train_loss": 0.503834640538251, - "val_score": 0.6029036769452016 - }, - { - "epoch": 10, - "test_score": 0.5764806679967731, - "train_loss": 0.499090200221097, - "val_score": 0.6023098264298107 - }, - { - "epoch": 11, - "test_score": 0.5769842279779496, - "train_loss": 0.4986710399389267, - "val_score": 0.6036243315348981 - }, - { - "epoch": 12, - "test_score": 0.5764756344426175, - "train_loss": 0.5011906392044492, - "val_score": 0.6016404639554996 - }, - { - "epoch": 13, - "test_score": 0.5759788406244468, - "train_loss": 0.5009944047088977, - "val_score": 0.6021128946314307 - }, - { - "epoch": 14, - "test_score": 0.5777112418096728, - "train_loss": 0.49901655702679243, - "val_score": 0.6042899349951487 - }, - { - "epoch": 15, - "test_score": 0.5778695632051555, - "train_loss": 0.4992442086890892, - "val_score": 0.6056838105765301 - }, - { - "epoch": 16, - "test_score": 0.5785535717635366, - "train_loss": 0.5002761604609313, - "val_score": 0.6056769964422184 - }, - { - "epoch": 17, - "test_score": 0.5816249076525681, - "train_loss": 0.4998697401196868, - "val_score": 0.6106538647226952 - }, - { - "epoch": 18, - "test_score": 0.580032497543074, - "train_loss": 0.498333211850237, - "val_score": 0.6061102338686076 - }, - { - "epoch": 19, - "test_score": 0.5802514540493597, - "train_loss": 0.498783348887055, - "val_score": 0.6063259275695186 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5816249076525681, - "score": 0.5816249076525681 - }, - "val": { - "roc-auc": 0.6106538647226952, - "score": 0.6106538647226952 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/mfb_only-evaluation/2", - "seed": 2, - "task_type": "binclass", - "time_seconds": 28.520503282546997, - "variant": "mfb_only" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0.toml deleted file mode 100644 index 27ba0bcca..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0.toml +++ /dev/null @@ -1,30 +0,0 @@ -dataset = "ecom-offers" -variant = "rla_only" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0/report.json deleted file mode 100644 index f53f8f9e8..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0/report.json +++ /dev/null @@ -1,179 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 19, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "rla_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5738058639740508, - "train_loss": 0.5600903784787213, - "val_score": 0.5996591097560906 - }, - { - "epoch": 1, - "test_score": 0.566052271996951, - "train_loss": 0.5203855821379909, - "val_score": 0.5867063728127937 - }, - { - "epoch": 2, - "test_score": 0.5697889358834675, - "train_loss": 0.5058877644715486, - "val_score": 0.5926538516914465 - }, - { - "epoch": 3, - "test_score": 0.5724541067249609, - "train_loss": 0.5015750681912458, - "val_score": 0.5966838501272544 - }, - { - "epoch": 4, - "test_score": 0.5748546719311031, - "train_loss": 0.4996969308014269, - "val_score": 0.6004681196027468 - }, - { - "epoch": 5, - "test_score": 0.5757011892813118, - "train_loss": 0.4983945345437085, - "val_score": 0.6016276869930021 - }, - { - "epoch": 6, - "test_score": 0.5777511104101054, - "train_loss": 0.49719943823637786, - "val_score": 0.6039715465042297 - }, - { - "epoch": 7, - "test_score": 0.5767656991997876, - "train_loss": 0.4959517259288717, - "val_score": 0.603024860924956 - }, - { - "epoch": 8, - "test_score": 0.576220767370786, - "train_loss": 0.4950623898594468, - "val_score": 0.6032534237210622 - }, - { - "epoch": 9, - "test_score": 0.5740754287760579, - "train_loss": 0.4947311133146286, - "val_score": 0.6016036108764757 - }, - { - "epoch": 10, - "test_score": 0.5750780606925876, - "train_loss": 0.4943293062625108, - "val_score": 0.60343813135313 - }, - { - "epoch": 11, - "test_score": 0.5747031631908137, - "train_loss": 0.4936874199796606, - "val_score": 0.6036642581607127 - }, - { - "epoch": 12, - "test_score": 0.5746027710608276, - "train_loss": 0.49342205844543596, - "val_score": 0.6047218457106929 - }, - { - "epoch": 13, - "test_score": 0.5753453008852248, - "train_loss": 0.49321615254437484, - "val_score": 0.6057724164344271 - }, - { - "epoch": 14, - "test_score": 0.5750546534259727, - "train_loss": 0.49275959255518736, - "val_score": 0.6064825457847545 - }, - { - "epoch": 15, - "test_score": 0.575529735392629, - "train_loss": 0.4924712076231285, - "val_score": 0.6070732601031206 - }, - { - "epoch": 16, - "test_score": 0.5750690164123993, - "train_loss": 0.492143577447644, - "val_score": 0.6063755060167295 - }, - { - "epoch": 17, - "test_score": 0.5759319579026756, - "train_loss": 0.4919159158512398, - "val_score": 0.6079652409718912 - }, - { - "epoch": 18, - "test_score": 0.5761580649073832, - "train_loss": 0.4918079514194418, - "val_score": 0.6090219661598224 - }, - { - "epoch": 19, - "test_score": 0.5766629173713693, - "train_loss": 0.49131932689083946, - "val_score": 0.6098132349342364 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5766629173713693, - "score": 0.5766629173713693 - }, - "val": { - "roc-auc": 0.6098132349342364, - "score": 0.6098132349342364 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/0", - "seed": 0, - "task_type": "binclass", - "time_seconds": 27.724525213241577, - "variant": "rla_only" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1.toml deleted file mode 100644 index c2afea846..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1.toml +++ /dev/null @@ -1,30 +0,0 @@ -dataset = "ecom-offers" -variant = "rla_only" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1/report.json deleted file mode 100644 index f477b0fc3..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1/report.json +++ /dev/null @@ -1,179 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 17, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "rla_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5744099679596921, - "train_loss": 0.5611426466041141, - "val_score": 0.6003674480467773 - }, - { - "epoch": 1, - "test_score": 0.5661563215123454, - "train_loss": 0.5209181766819071, - "val_score": 0.5867286357578321 - }, - { - "epoch": 2, - "test_score": 0.5714187226121498, - "train_loss": 0.5059594881755335, - "val_score": 0.5942213052750304 - }, - { - "epoch": 3, - "test_score": 0.5729606174102976, - "train_loss": 0.5019116169876523, - "val_score": 0.5964863655326891 - }, - { - "epoch": 4, - "test_score": 0.5735315166757, - "train_loss": 0.5001836750242445, - "val_score": 0.5971035735295656 - }, - { - "epoch": 5, - "test_score": 0.574901839804957, - "train_loss": 0.4984646196718569, - "val_score": 0.5994008654901384 - }, - { - "epoch": 6, - "test_score": 0.5767816801143355, - "train_loss": 0.4975106026287432, - "val_score": 0.6025049192595473 - }, - { - "epoch": 7, - "test_score": 0.5780280767683461, - "train_loss": 0.49668018243930956, - "val_score": 0.604435957950165 - }, - { - "epoch": 8, - "test_score": 0.5764066989266243, - "train_loss": 0.4956071581001635, - "val_score": 0.6016519326336919 - }, - { - "epoch": 9, - "test_score": 0.5754726646828595, - "train_loss": 0.4948139527329692, - "val_score": 0.6022930730200864 - }, - { - "epoch": 10, - "test_score": 0.5756381861684305, - "train_loss": 0.4942547689985346, - "val_score": 0.6035153864626877 - }, - { - "epoch": 11, - "test_score": 0.5749532849600514, - "train_loss": 0.49325044563523046, - "val_score": 0.6039946275876223 - }, - { - "epoch": 12, - "test_score": 0.574489857035036, - "train_loss": 0.493444146933379, - "val_score": 0.6038499866230694 - }, - { - "epoch": 13, - "test_score": 0.5753126292754014, - "train_loss": 0.4927494956387414, - "val_score": 0.6059532028988868 - }, - { - "epoch": 14, - "test_score": 0.5750983715795217, - "train_loss": 0.49215994499347826, - "val_score": 0.6051037983923124 - }, - { - "epoch": 15, - "test_score": 0.5755156389614102, - "train_loss": 0.492595124023932, - "val_score": 0.6076091812635939 - }, - { - "epoch": 16, - "test_score": 0.5749148452195115, - "train_loss": 0.49249838458167183, - "val_score": 0.6060914977632319 - }, - { - "epoch": 17, - "test_score": 0.5767242343675878, - "train_loss": 0.49214974487269364, - "val_score": 0.6088361897884629 - }, - { - "epoch": 18, - "test_score": 0.5751086773477281, - "train_loss": 0.4915305613367646, - "val_score": 0.6059502251701017 - }, - { - "epoch": 19, - "test_score": 0.5759139437297989, - "train_loss": 0.4917481326394611, - "val_score": 0.6069593914595525 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5767242343675878, - "score": 0.5767242343675878 - }, - "val": { - "roc-auc": 0.6088361897884629, - "score": 0.6088361897884629 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/1", - "seed": 1, - "task_type": "binclass", - "time_seconds": 28.521148443222046, - "variant": "rla_only" -} diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2.toml deleted file mode 100644 index 983c8f37c..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2.toml +++ /dev/null @@ -1,30 +0,0 @@ -dataset = "ecom-offers" -variant = "rla_only" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/ecom-offers" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2/report.json deleted file mode 100644 index dbef5ef90..000000000 --- a/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2/report.json +++ /dev/null @@ -1,179 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 19, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/ecom-offers" - }, - "dataset": "ecom-offers", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "rla_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2.toml", - "dataset": "ecom-offers", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.5682818730321174, - "train_loss": 0.5605212438989569, - "val_score": 0.5927119874236068 - }, - { - "epoch": 1, - "test_score": 0.5672713870353969, - "train_loss": 0.5217861185471216, - "val_score": 0.5892199739206975 - }, - { - "epoch": 2, - "test_score": 0.571274286883303, - "train_loss": 0.506144740515285, - "val_score": 0.5945899687363537 - }, - { - "epoch": 3, - "test_score": 0.5740664526844113, - "train_loss": 0.5019352408470931, - "val_score": 0.5979948099661399 - }, - { - "epoch": 4, - "test_score": 0.5757866233248725, - "train_loss": 0.4998747734008012, - "val_score": 0.6000116979043437 - }, - { - "epoch": 5, - "test_score": 0.5751457347205626, - "train_loss": 0.49779329476533113, - "val_score": 0.5992338436506943 - }, - { - "epoch": 6, - "test_score": 0.5762034660781161, - "train_loss": 0.4968773893736027, - "val_score": 0.6016685054633288 - }, - { - "epoch": 7, - "test_score": 0.5753777493325488, - "train_loss": 0.49569592983634386, - "val_score": 0.6005299443281173 - }, - { - "epoch": 8, - "test_score": 0.5747554111110105, - "train_loss": 0.49488269951608443, - "val_score": 0.5998391591589697 - }, - { - "epoch": 9, - "test_score": 0.5754841451536674, - "train_loss": 0.49462440113226575, - "val_score": 0.6011704324150131 - }, - { - "epoch": 10, - "test_score": 0.5759761843708069, - "train_loss": 0.4938305141749205, - "val_score": 0.6034632651530242 - }, - { - "epoch": 11, - "test_score": 0.5751711783449855, - "train_loss": 0.49337507177282264, - "val_score": 0.6030761346138018 - }, - { - "epoch": 12, - "test_score": 0.573761587914253, - "train_loss": 0.49311621652709114, - "val_score": 0.6010109138624582 - }, - { - "epoch": 13, - "test_score": 0.5745490849823007, - "train_loss": 0.4942300303114785, - "val_score": 0.6023989740279694 - }, - { - "epoch": 14, - "test_score": 0.5733634908109679, - "train_loss": 0.49301944121166513, - "val_score": 0.6001994532860468 - }, - { - "epoch": 15, - "test_score": 0.5757922860732975, - "train_loss": 0.49234316856772814, - "val_score": 0.6060068683525613 - }, - { - "epoch": 16, - "test_score": 0.5758632796434527, - "train_loss": 0.492086089319653, - "val_score": 0.6062278062466175 - }, - { - "epoch": 17, - "test_score": 0.5762892038706345, - "train_loss": 0.49177511422722425, - "val_score": 0.6073894079268349 - }, - { - "epoch": 18, - "test_score": 0.5743713763446734, - "train_loss": 0.49189673750488844, - "val_score": 0.6031209626991266 - }, - { - "epoch": 19, - "test_score": 0.5770335251939862, - "train_loss": 0.4917442583375507, - "val_score": 0.6081233738486749 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.5770335251939862, - "score": 0.5770335251939862 - }, - "val": { - "roc-auc": 0.6081233738486749, - "score": 0.6081233738486749 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/ecom-offers/rla_only-evaluation/2", - "seed": 2, - "task_type": "binclass", - "time_seconds": 30.517080783843994, - "variant": "rla_only" -} diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..dca12e41a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,147 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44650602179756516, + "mae": 0.34757497906684875, + "r2": 0.5420529246330261, + "score": -0.44650602179756516 + }, + "val": { + "rmse": 0.4652274815891859, + "mae": 0.3617282807826996, + "r2": 0.503640353679657, + "score": -0.4652274815891859 + }, + "test": { + "rmse": 0.4820660657381116, + "mae": 0.3761262893676758, + "r2": 0.48187607526779175, + "score": -0.4820660657381116 + } + }, + "time": "0:09:50.236903", + "diversity": { + "train": { + "mean_centered_corr": -0.020426616072654724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051189813762903214 + }, + "val": { + "mean_centered_corr": -0.02039431221783161, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05273108929395676 + }, + "test": { + "mean_centered_corr": -0.020482471212744713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053407639265060425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27 + ], + "head_selection_time": "0:00:00.028691" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/1/report.json new file mode 100644 index 000000000..f3a0fb9e6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/1/report.json @@ -0,0 +1,147 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.4486631357443507, + "mae": 0.3495190143585205, + "r2": 0.5376174449920654, + "score": -0.4486631357443507 + }, + "val": { + "rmse": 0.46461960653787665, + "mae": 0.3615076541900635, + "r2": 0.5049366354942322, + "score": -0.46461960653787665 + }, + "test": { + "rmse": 0.4822480664686614, + "mae": 0.37663528323173523, + "r2": 0.4814847707748413, + "score": -0.4822480664686614 + } + }, + "time": "0:08:23.758542", + "diversity": { + "train": { + "mean_centered_corr": -0.020497672259807587, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04839831218123436 + }, + "val": { + "mean_centered_corr": -0.020458761602640152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049754440784454346 + }, + "test": { + "mean_centered_corr": -0.02041211724281311, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05034574493765831 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 5 + ], + "head_selection_time": "0:00:00.027928" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/2/report.json new file mode 100644 index 000000000..0523e5aa6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-best-head-evaluation/2/report.json @@ -0,0 +1,147 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 28638, + "metrics": { + "train": { + "rmse": 0.4464703615247226, + "mae": 0.34800708293914795, + "r2": 0.5421260595321655, + "score": -0.4464703615247226 + }, + "val": { + "rmse": 0.46567157154030486, + "mae": 0.3625640869140625, + "r2": 0.5026922821998596, + "score": -0.46567157154030486 + }, + "test": { + "rmse": 0.48281151175783615, + "mae": 0.3768390119075775, + "r2": 0.48027241230010986, + "score": -0.48281151175783615 + } + }, + "time": "0:11:51.682809", + "diversity": { + "train": { + "mean_centered_corr": -0.020397048443555832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05753684788942337 + }, + "val": { + "mean_centered_corr": -0.02034812979400158, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059377871453762054 + }, + "test": { + "mean_centered_corr": -0.02046407200396061, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05992777645587921 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.029046" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..78c613dfb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..2c476e8cd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/0/report.json @@ -0,0 +1,143 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44389690465868414, + "mae": 0.34541288018226624, + "r2": 0.5473892688751221, + "score": -0.44389690465868414 + }, + "val": { + "rmse": 0.4627596029926704, + "mae": 0.35959550738334656, + "r2": 0.5088924169540405, + "score": -0.4627596029926704 + }, + "test": { + "rmse": 0.4802386655626689, + "mae": 0.37469902634620667, + "r2": 0.4857967495918274, + "score": -0.4802386655626689 + } + }, + "time": "0:09:50.236903", + "diversity": { + "train": { + "mean_centered_corr": -0.020426616072654724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051189813762903214 + }, + "val": { + "mean_centered_corr": -0.02039431221783161, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05273108929395676 + }, + "test": { + "mean_centered_corr": -0.020482471212744713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053407639265060425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml new file mode 100644 index 000000000..bc39632dd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/1/report.json new file mode 100644 index 000000000..97b04029e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/1/report.json @@ -0,0 +1,143 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.4455413230457775, + "mae": 0.34672579169273376, + "r2": 0.5440295934677124, + "score": -0.4455413230457775 + }, + "val": { + "rmse": 0.46274572430329897, + "mae": 0.35959213972091675, + "r2": 0.5089218616485596, + "score": -0.46274572430329897 + }, + "test": { + "rmse": 0.4804413950506519, + "mae": 0.3748830258846283, + "r2": 0.48536252975463867, + "score": -0.4804413950506519 + } + }, + "time": "0:08:23.758542", + "diversity": { + "train": { + "mean_centered_corr": -0.020497672259807587, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04839831218123436 + }, + "val": { + "mean_centered_corr": -0.020458761602640152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049754440784454346 + }, + "test": { + "mean_centered_corr": -0.02041211724281311, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05034574493765831 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml new file mode 100644 index 000000000..fc2b3587f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/2/report.json new file mode 100644 index 000000000..ef23f4c76 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-evaluation/2/report.json @@ -0,0 +1,143 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 28638, + "metrics": { + "train": { + "rmse": 0.4415416847338914, + "mae": 0.3435233533382416, + "r2": 0.5521793961524963, + "score": -0.4415416847338914 + }, + "val": { + "rmse": 0.4626209262038764, + "mae": 0.3592834174633026, + "r2": 0.5091867446899414, + "score": -0.4626209262038764 + }, + "test": { + "rmse": 0.48046188039443216, + "mae": 0.37472644448280334, + "r2": 0.485318660736084, + "score": -0.48046188039443216 + } + }, + "time": "0:11:51.682809", + "diversity": { + "train": { + "mean_centered_corr": -0.020397048443555832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05753684788942337 + }, + "val": { + "mean_centered_corr": -0.02034812979400158, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059377871453762054 + }, + "test": { + "mean_centered_corr": -0.02046407200396061, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05992777645587921 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..81f6192e3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44394242184625543, + "mae": 0.3455282151699066, + "r2": 0.547296404838562, + "score": -0.44394242184625543 + }, + "val": { + "rmse": 0.4625169879783321, + "mae": 0.3595449924468994, + "r2": 0.5094072818756104, + "score": -0.4625169879783321 + }, + "test": { + "rmse": 0.47990786370986477, + "mae": 0.3745970129966736, + "r2": 0.4865049123764038, + "score": -0.47990786370986477 + } + }, + "time": "0:09:50.236903", + "diversity": { + "train": { + "mean_centered_corr": -0.020426616072654724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051189813762903214 + }, + "val": { + "mean_centered_corr": -0.02039431221783161, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05273108929395676 + }, + "test": { + "mean_centered_corr": -0.020482471212744713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053407639265060425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27, + 17, + 5, + 3, + 14, + 4, + 22, + 6, + 24, + 10, + 16 + ], + "head_selection_time": "0:00:00.401983" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..cddf7150d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1/report.json @@ -0,0 +1,162 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.44558913020992225, + "mae": 0.34685587882995605, + "r2": 0.5439317226409912, + "score": -0.44558913020992225 + }, + "val": { + "rmse": 0.4624852364993238, + "mae": 0.3594231605529785, + "r2": 0.5094746351242065, + "score": -0.4624852364993238 + }, + "test": { + "rmse": 0.4800568652741734, + "mae": 0.3746061325073242, + "r2": 0.48618602752685547, + "score": -0.4800568652741734 + } + }, + "time": "0:08:23.758542", + "diversity": { + "train": { + "mean_centered_corr": -0.020497672259807587, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04839831218123436 + }, + "val": { + "mean_centered_corr": -0.020458761602640152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049754440784454346 + }, + "test": { + "mean_centered_corr": -0.02041211724281311, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05034574493765831 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 5, + 15, + 11, + 9, + 30, + 25, + 6, + 24, + 4, + 13, + 27, + 10, + 9, + 16, + 0, + 28 + ], + "head_selection_time": "0:00:00.658431" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..a9fa005f6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 28638, + "metrics": { + "train": { + "rmse": 0.4416625198883345, + "mae": 0.34376394748687744, + "r2": 0.5519342422485352, + "score": -0.4416625198883345 + }, + "val": { + "rmse": 0.46240787039558773, + "mae": 0.35923537611961365, + "r2": 0.5096387267112732, + "score": -0.46240787039558773 + }, + "test": { + "rmse": 0.480411417572094, + "mae": 0.3748789131641388, + "r2": 0.48542678356170654, + "score": -0.480411417572094 + } + }, + "time": "0:11:51.682809", + "diversity": { + "train": { + "mean_centered_corr": -0.020397048443555832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05753684788942337 + }, + "val": { + "mean_centered_corr": -0.02034812979400158, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059377871453762054 + }, + "test": { + "mean_centered_corr": -0.02046407200396061, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05992777645587921 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 13, + 25, + 19, + 23, + 6, + 29, + 7, + 31, + 21, + 3, + 2 + ], + "head_selection_time": "0:00:00.397654" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..222652a2b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/0/report.json @@ -0,0 +1,400 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.45199979641686133, + "mae": 0.3517935872077942, + "r2": 0.5307145118713379, + "score": -0.45199979641686133 + }, + "val": { + "rmse": 0.4638214410008231, + "mae": 0.360493928194046, + "r2": 0.5066360831260681, + "score": -0.4638214410008231 + }, + "test": { + "rmse": 0.4819659035779245, + "mae": 0.37611493468284607, + "r2": 0.48209136724472046, + "score": -0.4819659035779245 + } + }, + "time": "0:10:10.466567", + "diversity": { + "train": { + "mean_centered_corr": -0.005496397614479065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04279099032282829 + }, + "val": { + "mean_centered_corr": -0.005842795129865408, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04372712969779968 + }, + "test": { + "mean_centered_corr": -0.005908838007599115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044375959783792496 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.027383" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..62ae87d2c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/1/report.json @@ -0,0 +1,400 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 14652, + "metrics": { + "train": { + "rmse": 0.45153318527792236, + "mae": 0.351412832736969, + "r2": 0.5316829681396484, + "score": -0.45153318527792236 + }, + "val": { + "rmse": 0.4640342331314561, + "mae": 0.3607448935508728, + "r2": 0.5061832666397095, + "score": -0.4640342331314561 + }, + "test": { + "rmse": 0.4812057394258983, + "mae": 0.3755890727043152, + "r2": 0.4837237596511841, + "score": -0.4812057394258983 + } + }, + "time": "0:08:47.757971", + "diversity": { + "train": { + "mean_centered_corr": -0.011369073763489723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042549557983875275 + }, + "val": { + "mean_centered_corr": -0.011497393250465393, + "mean_pairwise_disagreement": NaN, + "member_std": 0.043487798422575 + }, + "test": { + "mean_centered_corr": -0.011657103896141052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0439562126994133 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.027507" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..78d8c3953 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-best-head-evaluation/2/report.json @@ -0,0 +1,400 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 12876, + "metrics": { + "train": { + "rmse": 0.452644751353011, + "mae": 0.35248246788978577, + "r2": 0.529374361038208, + "score": -0.452644751353011 + }, + "val": { + "rmse": 0.46399304740420017, + "mae": 0.3608178496360779, + "r2": 0.5062709450721741, + "score": -0.46399304740420017 + }, + "test": { + "rmse": 0.4807545037031361, + "mae": 0.3752909302711487, + "r2": 0.4846915602684021, + "score": -0.4807545037031361 + } + }, + "time": "0:07:48.322244", + "diversity": { + "train": { + "mean_centered_corr": -0.014557692222297192, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04135805740952492 + }, + "val": { + "mean_centered_corr": -0.014966246671974659, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042381998151540756 + }, + "test": { + "mean_centered_corr": -0.014708246104419231, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04306395351886749 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 23 + ], + "head_selection_time": "0:00:00.028302" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..784c17199 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..f58c2dd92 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,396 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4482607702587857, + "mae": 0.34888917207717896, + "r2": 0.5384464263916016, + "score": -0.4482607702587857 + }, + "val": { + "rmse": 0.4632019090821483, + "mae": 0.3600703775882721, + "r2": 0.5079531669616699, + "score": -0.4632019090821483 + }, + "test": { + "rmse": 0.4808496810468957, + "mae": 0.3752937614917755, + "r2": 0.48448747396469116, + "score": -0.4808496810468957 + } + }, + "time": "0:10:10.466567", + "diversity": { + "train": { + "mean_centered_corr": -0.005496397614479065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04279099032282829 + }, + "val": { + "mean_centered_corr": -0.005842795129865408, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04372712969779968 + }, + "test": { + "mean_centered_corr": -0.005908838007599115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044375959783792496 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml new file mode 100644 index 000000000..7ab1c4267 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1/report.json new file mode 100644 index 000000000..da30cf68e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1/report.json @@ -0,0 +1,396 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 14652, + "metrics": { + "train": { + "rmse": 0.449091304447225, + "mae": 0.3495481312274933, + "r2": 0.5367345213890076, + "score": -0.449091304447225 + }, + "val": { + "rmse": 0.4632797054631699, + "mae": 0.36018165946006775, + "r2": 0.5077878832817078, + "score": -0.4632797054631699 + }, + "test": { + "rmse": 0.48088074674610853, + "mae": 0.37531033158302307, + "r2": 0.48442089557647705, + "score": -0.48088074674610853 + } + }, + "time": "0:08:47.757971", + "diversity": { + "train": { + "mean_centered_corr": -0.011369073763489723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042549557983875275 + }, + "val": { + "mean_centered_corr": -0.011497393250465393, + "mean_pairwise_disagreement": NaN, + "member_std": 0.043487798422575 + }, + "test": { + "mean_centered_corr": -0.011657103896141052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0439562126994133 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml new file mode 100644 index 000000000..516923524 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2/report.json new file mode 100644 index 000000000..f9241e83b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2/report.json @@ -0,0 +1,396 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 12876, + "metrics": { + "train": { + "rmse": 0.4496297034631907, + "mae": 0.3498455882072449, + "r2": 0.5356230735778809, + "score": -0.4496297034631907 + }, + "val": { + "rmse": 0.46327719662519024, + "mae": 0.359968900680542, + "r2": 0.5077931880950928, + "score": -0.46327719662519024 + }, + "test": { + "rmse": 0.48070024325509664, + "mae": 0.3750893771648407, + "r2": 0.4848078489303589, + "score": -0.48070024325509664 + } + }, + "time": "0:07:48.322244", + "diversity": { + "train": { + "mean_centered_corr": -0.014557692222297192, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04135805740952492 + }, + "val": { + "mean_centered_corr": -0.014966246671974659, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042381998151540756 + }, + "test": { + "mean_centered_corr": -0.014708246104419231, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04306395351886749 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0f7f787a9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,413 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44782199154768965, + "mae": 0.34862202405929565, + "r2": 0.5393495559692383, + "score": -0.44782199154768965 + }, + "val": { + "rmse": 0.4628482268025946, + "mae": 0.35992974042892456, + "r2": 0.5087043046951294, + "score": -0.4628482268025946 + }, + "test": { + "rmse": 0.4803119031248436, + "mae": 0.3749502897262573, + "r2": 0.48563992977142334, + "score": -0.4803119031248436 + } + }, + "time": "0:10:10.466567", + "diversity": { + "train": { + "mean_centered_corr": -0.005496397614479065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04279099032282829 + }, + "val": { + "mean_centered_corr": -0.005842795129865408, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04372712969779968 + }, + "test": { + "mean_centered_corr": -0.005908838007599115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044375959783792496 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 27, + 14, + 2, + 21, + 25, + 2, + 12, + 22, + 14, + 27, + 2, + 19, + 24 + ], + "head_selection_time": "0:00:00.542336" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..5784ed8d0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,412 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 14652, + "metrics": { + "train": { + "rmse": 0.448658967573918, + "mae": 0.34933236241340637, + "r2": 0.5376260280609131, + "score": -0.448658967573918 + }, + "val": { + "rmse": 0.4629984701854322, + "mae": 0.36006447672843933, + "r2": 0.5083853006362915, + "score": -0.4629984701854322 + }, + "test": { + "rmse": 0.4803911006799199, + "mae": 0.375040739774704, + "r2": 0.4854702949523926, + "score": -0.4803911006799199 + } + }, + "time": "0:08:47.757971", + "diversity": { + "train": { + "mean_centered_corr": -0.011369073763489723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042549557983875275 + }, + "val": { + "mean_centered_corr": -0.011497393250465393, + "mean_pairwise_disagreement": NaN, + "member_std": 0.043487798422575 + }, + "test": { + "mean_centered_corr": -0.011657103896141052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0439562126994133 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 0, + 18, + 26, + 14, + 10, + 19, + 27, + 16, + 15, + 1, + 24, + 0 + ], + "head_selection_time": "0:00:00.482246" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..88ca26cfd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,410 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 12876, + "metrics": { + "train": { + "rmse": 0.4491811321927657, + "mae": 0.34958937764167786, + "r2": 0.5365492105484009, + "score": -0.4491811321927657 + }, + "val": { + "rmse": 0.46301623543457804, + "mae": 0.359862744808197, + "r2": 0.5083475708961487, + "score": -0.46301623543457804 + }, + "test": { + "rmse": 0.4803092971079454, + "mae": 0.3748699426651001, + "r2": 0.4856455326080322, + "score": -0.4803092971079454 + } + }, + "time": "0:07:48.322244", + "diversity": { + "train": { + "mean_centered_corr": -0.014557692222297192, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04135805740952492 + }, + "val": { + "mean_centered_corr": -0.014966246671974659, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042381998151540756 + }, + "test": { + "mean_centered_corr": -0.014708246104419231, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04306395351886749 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 23, + 2, + 11, + 22, + 15, + 0, + 12, + 25, + 18, + 8, + 21 + ], + "head_selection_time": "0:00:00.407629" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f1c458896 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/0/report.json @@ -0,0 +1,166 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44621279542701975, + "mae": 0.3473000228404999, + "r2": 0.5426542162895203, + "score": -0.44621279542701975 + }, + "val": { + "rmse": 0.46492255368357066, + "mae": 0.36141437292099, + "r2": 0.5042908191680908, + "score": -0.46492255368357066 + }, + "test": { + "rmse": 0.4825495500767423, + "mae": 0.37668848037719727, + "r2": 0.4808362126350403, + "score": -0.4825495500767423 + } + }, + "time": "0:37:31.095542", + "diversity": { + "train": { + "mean_centered_corr": -0.02036323957145214, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05055415257811546 + }, + "val": { + "mean_centered_corr": -0.02048061601817608, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05214421823620796 + }, + "test": { + "mean_centered_corr": -0.020323824137449265, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052737440913915634 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14 + ], + "head_selection_time": "0:00:00.027759" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/1/report.json new file mode 100644 index 000000000..07656ccee --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/1/report.json @@ -0,0 +1,166 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.44843356300247367, + "mae": 0.34928280115127563, + "r2": 0.5380905866622925, + "score": -0.44843356300247367 + }, + "val": { + "rmse": 0.46452431161307095, + "mae": 0.361534982919693, + "r2": 0.5051397085189819, + "score": -0.46452431161307095 + }, + "test": { + "rmse": 0.48248902135425475, + "mae": 0.37688297033309937, + "r2": 0.4809664487838745, + "score": -0.48248902135425475 + } + }, + "time": "0:32:03.279613", + "diversity": { + "train": { + "mean_centered_corr": -0.020463576540350914, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04766561836004257 + }, + "val": { + "mean_centered_corr": -0.020318932831287384, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048978473991155624 + }, + "test": { + "mean_centered_corr": -0.020468220114707947, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04966771602630615 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 5 + ], + "head_selection_time": "0:00:00.027757" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/2/report.json new file mode 100644 index 000000000..fa7f560b8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-best-head-evaluation/2/report.json @@ -0,0 +1,166 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 29526, + "metrics": { + "train": { + "rmse": 0.44484696398137763, + "mae": 0.3464239537715912, + "r2": 0.5454497337341309, + "score": -0.44484696398137763 + }, + "val": { + "rmse": 0.46544124766603506, + "mae": 0.3615325093269348, + "r2": 0.5031840801239014, + "score": -0.46544124766603506 + }, + "test": { + "rmse": 0.4832114117959526, + "mae": 0.37725383043289185, + "r2": 0.47941112518310547, + "score": -0.4832114117959526 + } + }, + "time": "0:46:36.662355", + "diversity": { + "train": { + "mean_centered_corr": -0.0203552283346653, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057090889662504196 + }, + "val": { + "mean_centered_corr": -0.020306019112467766, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058967314660549164 + }, + "test": { + "mean_centered_corr": -0.020541546866297722, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0595683790743351 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.027626" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/0.toml new file mode 100644 index 000000000..0b28045f1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/0/report.json new file mode 100644 index 000000000..9520ae39a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/0/report.json @@ -0,0 +1,162 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4437324533888703, + "mae": 0.3453030586242676, + "r2": 0.5477245450019836, + "score": -0.4437324533888703 + }, + "val": { + "rmse": 0.4627353230636857, + "mae": 0.35955438017845154, + "r2": 0.5089439749717712, + "score": -0.4627353230636857 + }, + "test": { + "rmse": 0.4803077303885999, + "mae": 0.3747372627258301, + "r2": 0.48564887046813965, + "score": -0.4803077303885999 + } + }, + "time": "0:37:31.095542", + "diversity": { + "train": { + "mean_centered_corr": -0.02036323957145214, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05055415257811546 + }, + "val": { + "mean_centered_corr": -0.02048061601817608, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05214421823620796 + }, + "test": { + "mean_centered_corr": -0.020323824137449265, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052737440913915634 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/1.toml new file mode 100644 index 000000000..a6c9a5d12 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/1.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/1/report.json new file mode 100644 index 000000000..374bdcb1a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/1/report.json @@ -0,0 +1,162 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.44556993438537174, + "mae": 0.34677401185035706, + "r2": 0.543971061706543, + "score": -0.44556993438537174 + }, + "val": { + "rmse": 0.4627920762114039, + "mae": 0.35961198806762695, + "r2": 0.5088235139846802, + "score": -0.4627920762114039 + }, + "test": { + "rmse": 0.4804951265119692, + "mae": 0.37489259243011475, + "r2": 0.4852474331855774, + "score": -0.4804951265119692 + } + }, + "time": "0:32:03.279613", + "diversity": { + "train": { + "mean_centered_corr": -0.020463576540350914, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04766561836004257 + }, + "val": { + "mean_centered_corr": -0.020318932831287384, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048978473991155624 + }, + "test": { + "mean_centered_corr": -0.020468220114707947, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04966771602630615 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/2.toml new file mode 100644 index 000000000..c040c0c45 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/2.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/2/report.json new file mode 100644 index 000000000..80a5dc885 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/2/report.json @@ -0,0 +1,162 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 29526, + "metrics": { + "train": { + "rmse": 0.44099685791198107, + "mae": 0.3432556390762329, + "r2": 0.5532838702201843, + "score": -0.44099685791198107 + }, + "val": { + "rmse": 0.4627156309124574, + "mae": 0.35954052209854126, + "r2": 0.5089857578277588, + "score": -0.4627156309124574 + }, + "test": { + "rmse": 0.4803808798726675, + "mae": 0.3748488426208496, + "r2": 0.48549216985702515, + "score": -0.4803808798726675 + } + }, + "time": "0:46:36.662355", + "diversity": { + "train": { + "mean_centered_corr": -0.0203552283346653, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057090889662504196 + }, + "val": { + "mean_centered_corr": -0.020306019112467766, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058967314660549164 + }, + "test": { + "mean_centered_corr": -0.020541546866297722, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0595683790743351 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..03c3c5cda --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4439349534537547, + "mae": 0.3455897867679596, + "r2": 0.5473116636276245, + "score": -0.4439349534537547 + }, + "val": { + "rmse": 0.4624234187671886, + "mae": 0.3594515919685364, + "r2": 0.5096057653427124, + "score": -0.4624234187671886 + }, + "test": { + "rmse": 0.4800586966547807, + "mae": 0.37484052777290344, + "r2": 0.4861820936203003, + "score": -0.4800586966547807 + } + }, + "time": "0:37:31.095542", + "diversity": { + "train": { + "mean_centered_corr": -0.02036323957145214, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05055415257811546 + }, + "val": { + "mean_centered_corr": -0.02048061601817608, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05214421823620796 + }, + "test": { + "mean_centered_corr": -0.020323824137449265, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052737440913915634 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14, + 3, + 17, + 24, + 5, + 10, + 4, + 27, + 31, + 7, + 17, + 3, + 24 + ], + "head_selection_time": "0:00:00.494060" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..bf8446d82 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1/report.json @@ -0,0 +1,180 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.44579561696398656, + "mae": 0.34707847237586975, + "r2": 0.5435090065002441, + "score": -0.44579561696398656 + }, + "val": { + "rmse": 0.46250972286520653, + "mae": 0.35948577523231506, + "r2": 0.5094226598739624, + "score": -0.46250972286520653 + }, + "test": { + "rmse": 0.4801020579816987, + "mae": 0.37471047043800354, + "r2": 0.486089289188385, + "score": -0.4801020579816987 + } + }, + "time": "0:32:03.279613", + "diversity": { + "train": { + "mean_centered_corr": -0.020463576540350914, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04766561836004257 + }, + "val": { + "mean_centered_corr": -0.020318932831287384, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048978473991155624 + }, + "test": { + "mean_centered_corr": -0.020468220114707947, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04966771602630615 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 5, + 13, + 9, + 6, + 25, + 30, + 15, + 24, + 14, + 8, + 13, + 3, + 4, + 9, + 27 + ], + "head_selection_time": "0:00:00.593938" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..e9da28fe3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 29526, + "metrics": { + "train": { + "rmse": 0.44166110285667354, + "mae": 0.3438146710395813, + "r2": 0.5519371628761292, + "score": -0.44166110285667354 + }, + "val": { + "rmse": 0.4623945128795896, + "mae": 0.3593231737613678, + "r2": 0.5096670389175415, + "score": -0.4623945128795896 + }, + "test": { + "rmse": 0.4799174115040007, + "mae": 0.37446993589401245, + "r2": 0.48648446798324585, + "score": -0.4799174115040007 + } + }, + "time": "0:46:36.662355", + "diversity": { + "train": { + "mean_centered_corr": -0.0203552283346653, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057090889662504196 + }, + "val": { + "mean_centered_corr": -0.020306019112467766, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058967314660549164 + }, + "test": { + "mean_centered_corr": -0.020541546866297722, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0595683790743351 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 21, + 12, + 5, + 25, + 16, + 7, + 29, + 0, + 8, + 22, + 26, + 3, + 2 + ], + "head_selection_time": "0:00:00.485946" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ef69ab721 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/0/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44801757104722606, + "mae": 0.348697692155838, + "r2": 0.5389471054077148, + "score": -0.44801757104722606 + }, + "val": { + "rmse": 0.46513258351328474, + "mae": 0.36148601770401, + "r2": 0.503842830657959, + "score": -0.46513258351328474 + }, + "test": { + "rmse": 0.48253251942068065, + "mae": 0.37657663226127625, + "r2": 0.48087286949157715, + "score": -0.48253251942068065 + } + }, + "time": "0:15:54.055312", + "diversity": { + "train": { + "mean_centered_corr": -0.020338764414191246, + "mean_pairwise_disagreement": NaN, + "member_std": 0.050871677696704865 + }, + "val": { + "mean_centered_corr": -0.02030899003148079, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05245138704776764 + }, + "test": { + "mean_centered_corr": -0.020395034924149513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053137533366680145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 7 + ], + "head_selection_time": "0:00:00.028441" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..635004bc0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/1/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.44870194270769953, + "mae": 0.349582701921463, + "r2": 0.5375374555587769, + "score": -0.44870194270769953 + }, + "val": { + "rmse": 0.46441974054703444, + "mae": 0.3615182936191559, + "r2": 0.5053625106811523, + "score": -0.46441974054703444 + }, + "test": { + "rmse": 0.48248829558115747, + "mae": 0.3769335150718689, + "r2": 0.48096799850463867, + "score": -0.48248829558115747 + } + }, + "time": "0:13:31.147489", + "diversity": { + "train": { + "mean_centered_corr": -0.02047574520111084, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0481378473341465 + }, + "val": { + "mean_centered_corr": -0.020433666184544563, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049497347325086594 + }, + "test": { + "mean_centered_corr": -0.02056153491139412, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05010291561484337 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 5 + ], + "head_selection_time": "0:00:00.028702" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..2f3742352 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-best-head-evaluation/2/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 29526, + "metrics": { + "train": { + "rmse": 0.44640390600457625, + "mae": 0.34810319542884827, + "r2": 0.5422623753547668, + "score": -0.44640390600457625 + }, + "val": { + "rmse": 0.4655328979544962, + "mae": 0.36213234066963196, + "r2": 0.5029884576797485, + "score": -0.4655328979544962 + }, + "test": { + "rmse": 0.48239242216697287, + "mae": 0.3769451081752777, + "r2": 0.4811742901802063, + "score": -0.48239242216697287 + } + }, + "time": "0:19:42.465328", + "diversity": { + "train": { + "mean_centered_corr": -0.02041659690439701, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05716099962592125 + }, + "val": { + "mean_centered_corr": -0.020310183987021446, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059111326932907104 + }, + "test": { + "mean_centered_corr": -0.02049497328698635, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05964430794119835 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 12 + ], + "head_selection_time": "0:00:00.027416" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml new file mode 100644 index 000000000..1d55eda06 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/0/report.json new file mode 100644 index 000000000..7b9d04d15 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/0/report.json @@ -0,0 +1,153 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4439953515264817, + "mae": 0.34551990032196045, + "r2": 0.547188401222229, + "score": -0.4439953515264817 + }, + "val": { + "rmse": 0.4627563346148938, + "mae": 0.3595946729183197, + "r2": 0.5088993906974792, + "score": -0.4627563346148938 + }, + "test": { + "rmse": 0.4802735094812688, + "mae": 0.3747326135635376, + "r2": 0.4857221841812134, + "score": -0.4802735094812688 + } + }, + "time": "0:15:54.055312", + "diversity": { + "train": { + "mean_centered_corr": -0.020338764414191246, + "mean_pairwise_disagreement": NaN, + "member_std": 0.050871677696704865 + }, + "val": { + "mean_centered_corr": -0.02030899003148079, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05245138704776764 + }, + "test": { + "mean_centered_corr": -0.020395034924149513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053137533366680145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml new file mode 100644 index 000000000..a63507c9a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/1/report.json new file mode 100644 index 000000000..6a6a08dbf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/1/report.json @@ -0,0 +1,153 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.4456571115584819, + "mae": 0.34683623909950256, + "r2": 0.5437926054000854, + "score": -0.4456571115584819 + }, + "val": { + "rmse": 0.4627833664641114, + "mae": 0.3596556782722473, + "r2": 0.5088419914245605, + "score": -0.4627833664641114 + }, + "test": { + "rmse": 0.4804732935000574, + "mae": 0.3749082386493683, + "r2": 0.4852942228317261, + "score": -0.4804732935000574 + } + }, + "time": "0:13:31.147489", + "diversity": { + "train": { + "mean_centered_corr": -0.02047574520111084, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0481378473341465 + }, + "val": { + "mean_centered_corr": -0.020433666184544563, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049497347325086594 + }, + "test": { + "mean_centered_corr": -0.02056153491139412, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05010291561484337 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml new file mode 100644 index 000000000..78470715d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/2/report.json new file mode 100644 index 000000000..ac2e0b2b8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/2/report.json @@ -0,0 +1,153 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 29526, + "metrics": { + "train": { + "rmse": 0.4413235510941761, + "mae": 0.3435209393501282, + "r2": 0.5526217818260193, + "score": -0.4413235510941761 + }, + "val": { + "rmse": 0.4626873390860138, + "mae": 0.359529584646225, + "r2": 0.5090458393096924, + "score": -0.4626873390860138 + }, + "test": { + "rmse": 0.4802976784438942, + "mae": 0.3747992217540741, + "r2": 0.48567038774490356, + "score": -0.4802976784438942 + } + }, + "time": "0:19:42.465328", + "diversity": { + "train": { + "mean_centered_corr": -0.02041659690439701, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05716099962592125 + }, + "val": { + "mean_centered_corr": -0.020310183987021446, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059111326932907104 + }, + "test": { + "mean_centered_corr": -0.02049497328698635, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05964430794119835 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d892e337e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4440075509733049, + "mae": 0.345533162355423, + "r2": 0.5471635460853577, + "score": -0.4440075509733049 + }, + "val": { + "rmse": 0.46246732196836127, + "mae": 0.35945001244544983, + "r2": 0.5095126032829285, + "score": -0.46246732196836127 + }, + "test": { + "rmse": 0.4800169922135662, + "mae": 0.3746195435523987, + "r2": 0.48627138137817383, + "score": -0.4800169922135662 + } + }, + "time": "0:15:54.055312", + "diversity": { + "train": { + "mean_centered_corr": -0.020338764414191246, + "mean_pairwise_disagreement": NaN, + "member_std": 0.050871677696704865 + }, + "val": { + "mean_centered_corr": -0.02030899003148079, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05245138704776764 + }, + "test": { + "mean_centered_corr": -0.020395034924149513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053137533366680145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 7, + 5, + 27, + 17, + 24, + 3, + 14, + 4, + 22, + 5, + 10, + 21, + 17, + 27, + 24, + 16, + 3, + 5, + 12, + 18, + 14, + 27, + 17 + ], + "head_selection_time": "0:00:01.091181" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..7f7238263 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,166 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.4459791714419137, + "mae": 0.3472432792186737, + "r2": 0.543133020401001, + "score": -0.4459791714419137 + }, + "val": { + "rmse": 0.46256428091435436, + "mae": 0.3595920205116272, + "r2": 0.5093069076538086, + "score": -0.46256428091435436 + }, + "test": { + "rmse": 0.4801574722160279, + "mae": 0.3747093081474304, + "r2": 0.4859706163406372, + "score": -0.4801574722160279 + } + }, + "time": "0:13:31.147489", + "diversity": { + "train": { + "mean_centered_corr": -0.02047574520111084, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0481378473341465 + }, + "val": { + "mean_centered_corr": -0.020433666184544563, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049497347325086594 + }, + "test": { + "mean_centered_corr": -0.02056153491139412, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05010291561484337 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 5, + 13, + 9, + 30, + 25, + 6, + 24, + 15, + 18, + 11 + ], + "head_selection_time": "0:00:00.355063" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..2ca396a30 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/cooking-time/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 29526, + "metrics": { + "train": { + "rmse": 0.44162036125193554, + "mae": 0.3438238501548767, + "r2": 0.5520198345184326, + "score": -0.44162036125193554 + }, + "val": { + "rmse": 0.462401054726952, + "mae": 0.3593364953994751, + "r2": 0.5096532106399536, + "score": -0.462401054726952 + }, + "test": { + "rmse": 0.48002014306639385, + "mae": 0.3746410012245178, + "r2": 0.48626458644866943, + "score": -0.48002014306639385 + } + }, + "time": "0:19:42.465328", + "diversity": { + "train": { + "mean_centered_corr": -0.02041659690439701, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05716099962592125 + }, + "val": { + "mean_centered_corr": -0.020310183987021446, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059111326932907104 + }, + "test": { + "mean_centered_corr": -0.02049497328698635, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05964430794119835 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 12, + 21, + 29, + 5, + 22, + 16, + 8, + 7, + 25, + 3, + 23, + 26, + 28, + 0, + 31, + 21, + 29, + 18 + ], + "head_selection_time": "0:00:00.787252" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..50ed2f8b2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/0/report.json @@ -0,0 +1,183 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8019205927848816, + "min_features_per_member": 141.0, + "mean_features_per_member": 153.96875, + "max_features_per_member": 167.0, + "min_members_per_feature": 21.0, + "mean_members_per_feature": 25.66145896911621, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6682771134235742, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44690438220657625, + "mae": 0.3479042053222656, + "r2": 0.5412354469299316, + "score": -0.44690438220657625 + }, + "val": { + "rmse": 0.4652213638497702, + "mae": 0.36172589659690857, + "r2": 0.5036534070968628, + "score": -0.4652213638497702 + }, + "test": { + "rmse": 0.48343941871247514, + "mae": 0.37721583247184753, + "r2": 0.4789196848869324, + "score": -0.48343941871247514 + } + }, + "time": "0:12:07.675037", + "diversity": { + "train": { + "mean_centered_corr": -0.019306395202875137, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057879555970430374 + }, + "val": { + "mean_centered_corr": -0.01832164078950882, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06112360581755638 + }, + "test": { + "mean_centered_corr": -0.01794510707259178, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06266573071479797 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.027886" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..4197d13eb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/1/report.json @@ -0,0 +1,183 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.8004557490348816, + "min_features_per_member": 142.0, + "mean_features_per_member": 153.6875, + "max_features_per_member": 165.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.61458396911621, + "max_members_per_feature": 30.0, + "mean_pairwise_jaccard": 0.6664211145724647, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 18870, + "metrics": { + "train": { + "rmse": 0.4492457506109391, + "mae": 0.3500923812389374, + "r2": 0.5364158153533936, + "score": -0.4492457506109391 + }, + "val": { + "rmse": 0.4651112788369628, + "mae": 0.3617507219314575, + "r2": 0.5038883090019226, + "score": -0.4651112788369628 + }, + "test": { + "rmse": 0.4825797035175412, + "mae": 0.3767949938774109, + "r2": 0.48077136278152466, + "score": -0.4825797035175412 + } + }, + "time": "0:10:14.470810", + "diversity": { + "train": { + "mean_centered_corr": -0.019374029710888863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05610343813896179 + }, + "val": { + "mean_centered_corr": -0.018558263778686523, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05900688096880913 + }, + "test": { + "mean_centered_corr": -0.018259482458233833, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06036880984902382 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 27 + ], + "head_selection_time": "0:00:00.027978" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..17110eb64 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-best-head-evaluation/2/report.json @@ -0,0 +1,183 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8037109375, + "min_features_per_member": 144.0, + "mean_features_per_member": 154.3125, + "max_features_per_member": 166.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.71875, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6710767481400324, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 28638, + "metrics": { + "train": { + "rmse": 0.44502781250652484, + "mae": 0.3467985689640045, + "r2": 0.5450800657272339, + "score": -0.44502781250652484 + }, + "val": { + "rmse": 0.4653975929915622, + "mae": 0.3622805178165436, + "r2": 0.5032773017883301, + "score": -0.4653975929915622 + }, + "test": { + "rmse": 0.48330849475451526, + "mae": 0.3774421513080597, + "r2": 0.47920191287994385, + "score": -0.48330849475451526 + } + }, + "time": "0:14:35.891802", + "diversity": { + "train": { + "mean_centered_corr": -0.019855137914419174, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06328506022691727 + }, + "val": { + "mean_centered_corr": -0.01927705854177475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06649371236562729 + }, + "test": { + "mean_centered_corr": -0.018950071185827255, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06782469898462296 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.028731" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml new file mode 100644 index 000000000..83cf7dea9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.8 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..835fe9a93 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8019205927848816, + "min_features_per_member": 141.0, + "mean_features_per_member": 153.96875, + "max_features_per_member": 167.0, + "min_members_per_feature": 21.0, + "mean_members_per_feature": 25.66145896911621, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6682771134235742, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4439757847261177, + "mae": 0.34554752707481384, + "r2": 0.5472283363342285, + "score": -0.4439757847261177 + }, + "val": { + "rmse": 0.4634085546239816, + "mae": 0.36031344532966614, + "r2": 0.5075141191482544, + "score": -0.4634085546239816 + }, + "test": { + "rmse": 0.4808915146871742, + "mae": 0.37534573674201965, + "r2": 0.4843977689743042, + "score": -0.4808915146871742 + } + }, + "time": "0:12:07.675037", + "diversity": { + "train": { + "mean_centered_corr": -0.019306395202875137, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057879555970430374 + }, + "val": { + "mean_centered_corr": -0.01832164078950882, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06112360581755638 + }, + "test": { + "mean_centered_corr": -0.01794510707259178, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06266573071479797 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml new file mode 100644 index 000000000..b98101f71 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.8 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 1 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/1/report.json new file mode 100644 index 000000000..e4dab74b3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/1/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.8004557490348816, + "min_features_per_member": 142.0, + "mean_features_per_member": 153.6875, + "max_features_per_member": 165.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.61458396911621, + "max_members_per_feature": 30.0, + "mean_pairwise_jaccard": 0.6664211145724647, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 18870, + "metrics": { + "train": { + "rmse": 0.4457924916226811, + "mae": 0.3470214605331421, + "r2": 0.5435153841972351, + "score": -0.4457924916226811 + }, + "val": { + "rmse": 0.4632980549483924, + "mae": 0.3602248728275299, + "r2": 0.5077489018440247, + "score": -0.4632980549483924 + }, + "test": { + "rmse": 0.48077915988352504, + "mae": 0.3752482533454895, + "r2": 0.48463869094848633, + "score": -0.48077915988352504 + } + }, + "time": "0:10:14.470810", + "diversity": { + "train": { + "mean_centered_corr": -0.019374029710888863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05610343813896179 + }, + "val": { + "mean_centered_corr": -0.018558263778686523, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05900688096880913 + }, + "test": { + "mean_centered_corr": -0.018259482458233833, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06036880984902382 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml new file mode 100644 index 000000000..308b77981 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.8 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 2 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/2/report.json new file mode 100644 index 000000000..c577c12b6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-evaluation/2/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8037109375, + "min_features_per_member": 144.0, + "mean_features_per_member": 154.3125, + "max_features_per_member": 166.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.71875, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6710767481400324, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 28638, + "metrics": { + "train": { + "rmse": 0.44161565421565047, + "mae": 0.34365126490592957, + "r2": 0.5520293712615967, + "score": -0.44161565421565047 + }, + "val": { + "rmse": 0.4630663413507467, + "mae": 0.35984113812446594, + "r2": 0.5082411766052246, + "score": -0.4630663413507467 + }, + "test": { + "rmse": 0.48078120546833536, + "mae": 0.3750513195991516, + "r2": 0.48463428020477295, + "score": -0.48078120546833536 + } + }, + "time": "0:14:35.891802", + "diversity": { + "train": { + "mean_centered_corr": -0.019855137914419174, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06328506022691727 + }, + "val": { + "mean_centered_corr": -0.01927705854177475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06649371236562729 + }, + "test": { + "mean_centered_corr": -0.018950071185827255, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06782469898462296 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c9f5dc71a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,193 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8019205927848816, + "min_features_per_member": 141.0, + "mean_features_per_member": 153.96875, + "max_features_per_member": 167.0, + "min_members_per_feature": 21.0, + "mean_members_per_feature": 25.66145896911621, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6682771134235742, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44390037903332324, + "mae": 0.34550046920776367, + "r2": 0.547382116317749, + "score": -0.44390037903332324 + }, + "val": { + "rmse": 0.4624822561623469, + "mae": 0.35955899953842163, + "r2": 0.5094809532165527, + "score": -0.4624822561623469 + }, + "test": { + "rmse": 0.4798057758257153, + "mae": 0.37443193793296814, + "r2": 0.4867233633995056, + "score": -0.4798057758257153 + } + }, + "time": "0:12:07.675037", + "diversity": { + "train": { + "mean_centered_corr": -0.019306395202875137, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057879555970430374 + }, + "val": { + "mean_centered_corr": -0.01832164078950882, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06112360581755638 + }, + "test": { + "mean_centered_corr": -0.01794510707259178, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06266573071479797 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 19, + 27, + 17, + 24, + 25, + 20, + 22, + 4, + 14, + 7, + 5 + ], + "head_selection_time": "0:00:00.407380" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..84931fdf6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,202 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.8004557490348816, + "min_features_per_member": 142.0, + "mean_features_per_member": 153.6875, + "max_features_per_member": 165.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.61458396911621, + "max_members_per_feature": 30.0, + "mean_pairwise_jaccard": 0.6664211145724647, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 18870, + "metrics": { + "train": { + "rmse": 0.44578164467993003, + "mae": 0.34703874588012695, + "r2": 0.5435376167297363, + "score": -0.44578164467993003 + }, + "val": { + "rmse": 0.4626697544496206, + "mae": 0.3596174120903015, + "r2": 0.5090831518173218, + "score": -0.4626697544496206 + }, + "test": { + "rmse": 0.47992630708272743, + "mae": 0.37454068660736084, + "r2": 0.4864654541015625, + "score": -0.47992630708272743 + } + }, + "time": "0:10:14.470810", + "diversity": { + "train": { + "mean_centered_corr": -0.019374029710888863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05610343813896179 + }, + "val": { + "mean_centered_corr": -0.018558263778686523, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05900688096880913 + }, + "test": { + "mean_centered_corr": -0.018259482458233833, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06036880984902382 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 27, + 10, + 18, + 4, + 2, + 23, + 28, + 6, + 24, + 11, + 22, + 20, + 25, + 10, + 19, + 30, + 28, + 18, + 15, + 9 + ], + "head_selection_time": "0:00:00.867913" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..d99ab4966 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,193 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8037109375, + "min_features_per_member": 144.0, + "mean_features_per_member": 154.3125, + "max_features_per_member": 166.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.71875, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6710767481400324, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 28638, + "metrics": { + "train": { + "rmse": 0.44202025927421124, + "mae": 0.34406524896621704, + "r2": 0.5512081384658813, + "score": -0.44202025927421124 + }, + "val": { + "rmse": 0.462357531977379, + "mae": 0.3593025505542755, + "r2": 0.5097454786300659, + "score": -0.462357531977379 + }, + "test": { + "rmse": 0.48018644149208806, + "mae": 0.37463948130607605, + "r2": 0.485908567905426, + "score": -0.48018644149208806 + } + }, + "time": "0:14:35.891802", + "diversity": { + "train": { + "mean_centered_corr": -0.019855137914419174, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06328506022691727 + }, + "val": { + "mean_centered_corr": -0.01927705854177475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06649371236562729 + }, + "test": { + "mean_centered_corr": -0.018950071185827255, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06782469898462296 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 19, + 13, + 29, + 15, + 7, + 8, + 24, + 13, + 17, + 26, + 30 + ], + "head_selection_time": "0:00:00.396628" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8abb38b47 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/0/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4475775717786638, + "mae": 0.3483501672744751, + "r2": 0.5398522615432739, + "score": -0.4475775717786638 + }, + "val": { + "rmse": 0.4649795846661889, + "mae": 0.361278235912323, + "r2": 0.504169225692749, + "score": -0.4649795846661889 + }, + "test": { + "rmse": 0.4827695666979872, + "mae": 0.3767538070678711, + "r2": 0.4803627133369446, + "score": -0.4827695666979872 + } + }, + "time": "0:23:06.357271", + "diversity": { + "train": { + "mean_centered_corr": -0.02037157490849495, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513317808508873 + }, + "val": { + "mean_centered_corr": -0.020479049533605576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05293743312358856 + }, + "test": { + "mean_centered_corr": -0.02046302892267704, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05355745181441307 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7 + ], + "head_selection_time": "0:00:00.028943" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..4e96471bb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/1/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.4483249230582749, + "mae": 0.34914642572402954, + "r2": 0.5383143424987793, + "score": -0.4483249230582749 + }, + "val": { + "rmse": 0.46454139300633795, + "mae": 0.3614574670791626, + "r2": 0.5051032900810242, + "score": -0.46454139300633795 + }, + "test": { + "rmse": 0.4826743974666991, + "mae": 0.37691187858581543, + "r2": 0.4805675745010376, + "score": -0.4826743974666991 + } + }, + "time": "0:19:42.335963", + "diversity": { + "train": { + "mean_centered_corr": -0.020455842837691307, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048325903713703156 + }, + "val": { + "mean_centered_corr": -0.020279740914702415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04973817616701126 + }, + "test": { + "mean_centered_corr": -0.02030608057975769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05035385116934776 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 5 + ], + "head_selection_time": "0:00:00.028060" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..c1662c3e5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-best-head-evaluation/2/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 28638, + "metrics": { + "train": { + "rmse": 0.44586212972829725, + "mae": 0.34740886092185974, + "r2": 0.5433727502822876, + "score": -0.44586212972829725 + }, + "val": { + "rmse": 0.46551838172590837, + "mae": 0.3620719611644745, + "r2": 0.5030194520950317, + "score": -0.46551838172590837 + }, + "test": { + "rmse": 0.4845241655350321, + "mae": 0.3784400224685669, + "r2": 0.47657865285873413, + "score": -0.4845241655350321 + } + }, + "time": "0:27:52.069713", + "diversity": { + "train": { + "mean_centered_corr": -0.020423956215381622, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05755901709198952 + }, + "val": { + "mean_centered_corr": -0.020522931590676308, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05942666903138161 + }, + "test": { + "mean_centered_corr": -0.02022053673863411, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05996261537075043 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.027846" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml new file mode 100644 index 000000000..364d0e3c9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/0/report.json new file mode 100644 index 000000000..35053ecb2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/0/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4435449287553804, + "mae": 0.3451736271381378, + "r2": 0.5481066703796387, + "score": -0.4435449287553804 + }, + "val": { + "rmse": 0.46272482498106277, + "mae": 0.3595674932003021, + "r2": 0.5089662671089172, + "score": -0.46272482498106277 + }, + "test": { + "rmse": 0.4802920784266626, + "mae": 0.37473270297050476, + "r2": 0.4856823682785034, + "score": -0.4802920784266626 + } + }, + "time": "0:23:06.357271", + "diversity": { + "train": { + "mean_centered_corr": -0.02037157490849495, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513317808508873 + }, + "val": { + "mean_centered_corr": -0.020479049533605576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05293743312358856 + }, + "test": { + "mean_centered_corr": -0.02046302892267704, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05355745181441307 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml new file mode 100644 index 000000000..647925031 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/1/report.json new file mode 100644 index 000000000..43edb6098 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/1/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.4453989580783725, + "mae": 0.34662750363349915, + "r2": 0.5443209409713745, + "score": -0.4453989580783725 + }, + "val": { + "rmse": 0.4627833825636106, + "mae": 0.35961589217185974, + "r2": 0.5088419914245605, + "score": -0.4627833825636106 + }, + "test": { + "rmse": 0.4804816670736117, + "mae": 0.3748900294303894, + "r2": 0.4852762818336487, + "score": -0.4804816670736117 + } + }, + "time": "0:19:42.335963", + "diversity": { + "train": { + "mean_centered_corr": -0.020455842837691307, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048325903713703156 + }, + "val": { + "mean_centered_corr": -0.020279740914702415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04973817616701126 + }, + "test": { + "mean_centered_corr": -0.02030608057975769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05035385116934776 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml new file mode 100644 index 000000000..6c85aa21c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/2/report.json new file mode 100644 index 000000000..605a6aa11 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-evaluation/2/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 28638, + "metrics": { + "train": { + "rmse": 0.44116400088657864, + "mae": 0.3432346284389496, + "r2": 0.5529451370239258, + "score": -0.44116400088657864 + }, + "val": { + "rmse": 0.4626829107835815, + "mae": 0.35933375358581543, + "r2": 0.5090552568435669, + "score": -0.4626829107835815 + }, + "test": { + "rmse": 0.4805534877331418, + "mae": 0.37477874755859375, + "r2": 0.4851223826408386, + "score": -0.4805534877331418 + } + }, + "time": "0:27:52.069713", + "diversity": { + "train": { + "mean_centered_corr": -0.020423956215381622, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05755901709198952 + }, + "val": { + "mean_centered_corr": -0.020522931590676308, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05942666903138161 + }, + "test": { + "mean_centered_corr": -0.02022053673863411, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05996261537075043 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..298a7858c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44362437525125054, + "mae": 0.3453079164028168, + "r2": 0.5479447841644287, + "score": -0.44362437525125054 + }, + "val": { + "rmse": 0.4624107384246987, + "mae": 0.35940805077552795, + "r2": 0.5096326470375061, + "score": -0.4624107384246987 + }, + "test": { + "rmse": 0.479974942646056, + "mae": 0.37463974952697754, + "r2": 0.48636138439178467, + "score": -0.479974942646056 + } + }, + "time": "0:23:06.357271", + "diversity": { + "train": { + "mean_centered_corr": -0.02037157490849495, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513317808508873 + }, + "val": { + "mean_centered_corr": -0.020479049533605576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05293743312358856 + }, + "test": { + "mean_centered_corr": -0.02046302892267704, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05355745181441307 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7, + 31, + 14, + 17, + 3, + 4, + 5, + 24, + 27, + 22, + 10, + 6, + 14, + 31, + 17, + 3, + 5, + 16, + 24, + 4, + 9, + 27, + 14 + ], + "head_selection_time": "0:00:01.063797" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..d7097eada --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,169 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 19092, + "metrics": { + "train": { + "rmse": 0.4455662222054085, + "mae": 0.34683141112327576, + "r2": 0.5439786911010742, + "score": -0.4455662222054085 + }, + "val": { + "rmse": 0.46252858614858405, + "mae": 0.359448105096817, + "r2": 0.5093826651573181, + "score": -0.46252858614858405 + }, + "test": { + "rmse": 0.48029415711103896, + "mae": 0.37478509545326233, + "r2": 0.48567795753479004, + "score": -0.48029415711103896 + } + }, + "time": "0:19:42.335963", + "diversity": { + "train": { + "mean_centered_corr": -0.020455842837691307, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048325903713703156 + }, + "val": { + "mean_centered_corr": -0.020279740914702415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04973817616701126 + }, + "test": { + "mean_centered_corr": -0.02030608057975769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05035385116934776 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 5, + 25, + 15, + 13, + 8, + 30, + 24, + 6, + 9, + 4, + 25, + 15, + 13, + 27 + ], + "head_selection_time": "0:00:00.544937" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..fbb58df48 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 28638, + "metrics": { + "train": { + "rmse": 0.4413632397247652, + "mae": 0.3435739278793335, + "r2": 0.5525412559509277, + "score": -0.4413632397247652 + }, + "val": { + "rmse": 0.462385779530238, + "mae": 0.3592732548713684, + "r2": 0.5096855759620667, + "score": -0.462385779530238 + }, + "test": { + "rmse": 0.4801696683856488, + "mae": 0.3746996223926544, + "r2": 0.4859445095062256, + "score": -0.4801696683856488 + } + }, + "time": "0:27:52.069713", + "diversity": { + "train": { + "mean_centered_corr": -0.020423956215381622, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05755901709198952 + }, + "val": { + "mean_centered_corr": -0.020522931590676308, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05942666903138161 + }, + "test": { + "mean_centered_corr": -0.02022053673863411, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05996261537075043 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 25, + 13, + 21, + 19, + 6, + 7, + 1, + 31, + 15, + 23, + 27, + 30, + 13, + 14, + 5, + 19, + 21, + 22, + 13, + 29, + 6, + 7, + 2, + 31 + ], + "head_selection_time": "0:00:01.133563" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..55ac8b264 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,168 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:16.600998", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023548" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/1/report.json new file mode 100644 index 000000000..146ca13ec --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/1/report.json @@ -0,0 +1,168 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5157048423679549, + "mae": 0.408806711435318, + "r2": 0.5015189051628113, + "score": -0.5157048423679549 + }, + "val": { + "rmse": 0.5578575885405295, + "mae": 0.44588109850883484, + "r2": 0.41315340995788574, + "score": -0.5578575885405295 + }, + "test": { + "rmse": 0.5568376961917461, + "mae": 0.4522704780101776, + "r2": 0.4222150444984436, + "score": -0.5568376961917461 + } + }, + "time": "0:06:43.806101", + "diversity": { + "train": { + "mean_centered_corr": -0.02029341645538807, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06168676167726517 + }, + "val": { + "mean_centered_corr": -0.020232515409588814, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0631292313337326 + }, + "test": { + "mean_centered_corr": -0.020325176417827606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061680618673563004 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.023481" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/2/report.json new file mode 100644 index 000000000..708292a23 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/2/report.json @@ -0,0 +1,168 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5148308736438392, + "mae": 0.40706372261047363, + "r2": 0.5032070279121399, + "score": -0.5148308736438392 + }, + "val": { + "rmse": 0.5576450053690213, + "mae": 0.4445996880531311, + "r2": 0.4136006236076355, + "score": -0.5576450053690213 + }, + "test": { + "rmse": 0.5574656479348947, + "mae": 0.45258787274360657, + "r2": 0.42091113328933716, + "score": -0.5574656479348947 + } + }, + "time": "0:06:38.793392", + "diversity": { + "train": { + "mean_centered_corr": -0.0205259807407856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06119334325194359 + }, + "val": { + "mean_centered_corr": -0.020698808133602142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06522355228662491 + }, + "test": { + "mean_centered_corr": -0.020667163655161858, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06273415684700012 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.023044" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..d07370fc8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..5bbb360c8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/0/report.json @@ -0,0 +1,164 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:16.600998", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml new file mode 100644 index 000000000..0e316c4c0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/1/report.json new file mode 100644 index 000000000..2147e5dc2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/1/report.json @@ -0,0 +1,164 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5105330146478386, + "mae": 0.40035760402679443, + "r2": 0.5114669799804688, + "score": -0.5105330146478386 + }, + "val": { + "rmse": 0.5562247345576334, + "mae": 0.439607709646225, + "r2": 0.416583776473999, + "score": -0.5562247345576334 + }, + "test": { + "rmse": 0.550284872190118, + "mae": 0.44156232476234436, + "r2": 0.4357336759567261, + "score": -0.550284872190118 + } + }, + "time": "0:06:43.806101", + "diversity": { + "train": { + "mean_centered_corr": -0.02029341645538807, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06168676167726517 + }, + "val": { + "mean_centered_corr": -0.020232515409588814, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0631292313337326 + }, + "test": { + "mean_centered_corr": -0.020325176417827606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061680618673563004 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml new file mode 100644 index 000000000..5d2c3b3bb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/2/report.json new file mode 100644 index 000000000..2b292662d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-evaluation/2/report.json @@ -0,0 +1,164 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5116329266744724, + "mae": 0.4038048982620239, + "r2": 0.5093597173690796, + "score": -0.5116329266744724 + }, + "val": { + "rmse": 0.5557493871736087, + "mae": 0.4417022466659546, + "r2": 0.4175804853439331, + "score": -0.5557493871736087 + }, + "test": { + "rmse": 0.5507763801521746, + "mae": 0.44514498114585876, + "r2": 0.43472522497177124, + "score": -0.5507763801521746 + } + }, + "time": "0:06:38.793392", + "diversity": { + "train": { + "mean_centered_corr": -0.0205259807407856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06119334325194359 + }, + "val": { + "mean_centered_corr": -0.020698808133602142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06522355228662491 + }, + "test": { + "mean_centered_corr": -0.020667163655161858, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06273415684700012 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..fc04b59cb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,181 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:16.600998", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.432507" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..ecd089b66 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1/report.json @@ -0,0 +1,176 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5108096659852551, + "mae": 0.4014206528663635, + "r2": 0.5109373331069946, + "score": -0.5108096659852551 + }, + "val": { + "rmse": 0.5553836483690917, + "mae": 0.440256267786026, + "r2": 0.4183468818664551, + "score": -0.5553836483690917 + }, + "test": { + "rmse": 0.5515298017800222, + "mae": 0.4441634714603424, + "r2": 0.433177649974823, + "score": -0.5515298017800222 + } + }, + "time": "0:06:43.806101", + "diversity": { + "train": { + "mean_centered_corr": -0.02029341645538807, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06168676167726517 + }, + "val": { + "mean_centered_corr": -0.020232515409588814, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0631292313337326 + }, + "test": { + "mean_centered_corr": -0.020325176417827606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061680618673563004 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 22, + 30, + 29, + 19, + 11, + 18, + 30, + 4, + 16 + ], + "head_selection_time": "0:00:00.255273" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..5cc033c90 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2/report.json @@ -0,0 +1,176 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5120909698461144, + "mae": 0.4043644666671753, + "r2": 0.5084807872772217, + "score": -0.5120909698461144 + }, + "val": { + "rmse": 0.5554437183690072, + "mae": 0.4415339529514313, + "r2": 0.41822099685668945, + "score": -0.5554437183690072 + }, + "test": { + "rmse": 0.5516331354574737, + "mae": 0.44613465666770935, + "r2": 0.4329652190208435, + "score": -0.5516331354574737 + } + }, + "time": "0:06:38.793392", + "diversity": { + "train": { + "mean_centered_corr": -0.0205259807407856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06119334325194359 + }, + "val": { + "mean_centered_corr": -0.020698808133602142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06522355228662491 + }, + "test": { + "mean_centered_corr": -0.020667163655161858, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06273415684700012 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 8, + 31, + 22, + 14, + 12, + 0, + 16, + 27, + 13 + ], + "head_selection_time": "0:00:00.254917" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0927cc2e8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/0/report.json @@ -0,0 +1,449 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5140202432442316, + "mae": 0.40386128425598145, + "r2": 0.5047702789306641, + "score": -0.5140202432442316 + }, + "val": { + "rmse": 0.557661572508718, + "mae": 0.44313740730285645, + "r2": 0.4135657548904419, + "score": -0.557661572508718 + }, + "test": { + "rmse": 0.553144673358601, + "mae": 0.44622060656547546, + "r2": 0.42985349893569946, + "score": -0.553144673358601 + } + }, + "time": "0:08:00.261340", + "diversity": { + "train": { + "mean_centered_corr": -0.01827983371913433, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0643538236618042 + }, + "val": { + "mean_centered_corr": -0.018152762204408646, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06669643521308899 + }, + "test": { + "mean_centered_corr": -0.018407996743917465, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06605677306652069 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 30 + ], + "head_selection_time": "0:00:00.023441" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..4a2e2847e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/1/report.json @@ -0,0 +1,449 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5190214940362947, + "mae": 0.4106391668319702, + "r2": 0.49508655071258545, + "score": -0.5190214940362947 + }, + "val": { + "rmse": 0.5570080264093609, + "mae": 0.441851943731308, + "r2": 0.4149394631385803, + "score": -0.5570080264093609 + }, + "test": { + "rmse": 0.5540194450901057, + "mae": 0.44831377267837524, + "r2": 0.4280487298965454, + "score": -0.5540194450901057 + } + }, + "time": "0:07:03.676796", + "diversity": { + "train": { + "mean_centered_corr": -0.0186633188277483, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056279826909303665 + }, + "val": { + "mean_centered_corr": -0.018989257514476776, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05482005700469017 + }, + "test": { + "mean_centered_corr": -0.019170163199305534, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05448836833238602 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 30 + ], + "head_selection_time": "0:00:00.023472" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..98f0c74f5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/2/report.json @@ -0,0 +1,449 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5183413941542616, + "mae": 0.4073716104030609, + "r2": 0.4964088797569275, + "score": -0.5183413941542616 + }, + "val": { + "rmse": 0.5578165315968546, + "mae": 0.4420972168445587, + "r2": 0.4132397770881653, + "score": -0.5578165315968546 + }, + "test": { + "rmse": 0.5527351028922852, + "mae": 0.44726672768592834, + "r2": 0.430697500705719, + "score": -0.5527351028922852 + } + }, + "time": "0:07:03.426340", + "diversity": { + "train": { + "mean_centered_corr": -0.018950974568724632, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060403864830732346 + }, + "val": { + "mean_centered_corr": -0.019536394625902176, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06044872850179672 + }, + "test": { + "mean_centered_corr": -0.01978791505098343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06026185676455498 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.023048" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..f7bf33592 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..8dc921357 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,445 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5080552327962835, + "mae": 0.3986222445964813, + "r2": 0.5161974430084229, + "score": -0.5080552327962835 + }, + "val": { + "rmse": 0.5562445854609275, + "mae": 0.4416752755641937, + "r2": 0.4165421724319458, + "score": -0.5562445854609275 + }, + "test": { + "rmse": 0.5504166498793033, + "mae": 0.44359084963798523, + "r2": 0.4354633688926697, + "score": -0.5504166498793033 + } + }, + "time": "0:08:00.261340", + "diversity": { + "train": { + "mean_centered_corr": -0.01827983371913433, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0643538236618042 + }, + "val": { + "mean_centered_corr": -0.018152762204408646, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06669643521308899 + }, + "test": { + "mean_centered_corr": -0.018407996743917465, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06605677306652069 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml new file mode 100644 index 000000000..18508bd65 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1/report.json new file mode 100644 index 000000000..3e98b4d75 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1/report.json @@ -0,0 +1,445 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5155687012426072, + "mae": 0.40611687302589417, + "r2": 0.5017820596694946, + "score": -0.5155687012426072 + }, + "val": { + "rmse": 0.5564364677950075, + "mae": 0.44130873680114746, + "r2": 0.41613954305648804, + "score": -0.5564364677950075 + }, + "test": { + "rmse": 0.550973168882935, + "mae": 0.4452160894870758, + "r2": 0.43432122468948364, + "score": -0.550973168882935 + } + }, + "time": "0:07:03.676796", + "diversity": { + "train": { + "mean_centered_corr": -0.0186633188277483, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056279826909303665 + }, + "val": { + "mean_centered_corr": -0.018989257514476776, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05482005700469017 + }, + "test": { + "mean_centered_corr": -0.019170163199305534, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05448836833238602 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml new file mode 100644 index 000000000..094fb6a0f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2/report.json new file mode 100644 index 000000000..6851f0002 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2/report.json @@ -0,0 +1,445 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.514369679188237, + "mae": 0.4027121365070343, + "r2": 0.5040967464447021, + "score": -0.514369679188237 + }, + "val": { + "rmse": 0.5569470014167281, + "mae": 0.43902793526649475, + "r2": 0.4150676727294922, + "score": -0.5569470014167281 + }, + "test": { + "rmse": 0.5474489469980929, + "mae": 0.43932023644447327, + "r2": 0.4415346384048462, + "score": -0.5474489469980929 + } + }, + "time": "0:07:03.426340", + "diversity": { + "train": { + "mean_centered_corr": -0.018950974568724632, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060403864830732346 + }, + "val": { + "mean_centered_corr": -0.019536394625902176, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06044872850179672 + }, + "test": { + "mean_centered_corr": -0.01978791505098343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06026185676455498 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..e627e5168 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,457 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5096923366202544, + "mae": 0.4003257751464844, + "r2": 0.5130745768547058, + "score": -0.5096923366202544 + }, + "val": { + "rmse": 0.5556691040453957, + "mae": 0.4418928027153015, + "r2": 0.4177488088607788, + "score": -0.5556691040453957 + }, + "test": { + "rmse": 0.5521822756750591, + "mae": 0.44639116525650024, + "r2": 0.43183577060699463, + "score": -0.5521822756750591 + } + }, + "time": "0:08:00.261340", + "diversity": { + "train": { + "mean_centered_corr": -0.01827983371913433, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0643538236618042 + }, + "val": { + "mean_centered_corr": -0.018152762204408646, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06669643521308899 + }, + "test": { + "mean_centered_corr": -0.018407996743917465, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06605677306652069 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 30, + 24, + 27, + 15, + 4, + 21, + 24, + 1, + 27 + ], + "head_selection_time": "0:00:00.261765" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..49059cfd7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,458 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.51574838491419, + "mae": 0.4075404703617096, + "r2": 0.5014347434043884, + "score": -0.51574838491419 + }, + "val": { + "rmse": 0.5559175580175352, + "mae": 0.4421222507953644, + "r2": 0.41722798347473145, + "score": -0.5559175580175352 + }, + "test": { + "rmse": 0.5539339615490284, + "mae": 0.4493137001991272, + "r2": 0.4282252788543701, + "score": -0.5539339615490284 + } + }, + "time": "0:07:03.676796", + "diversity": { + "train": { + "mean_centered_corr": -0.0186633188277483, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056279826909303665 + }, + "val": { + "mean_centered_corr": -0.018989257514476776, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05482005700469017 + }, + "test": { + "mean_centered_corr": -0.019170163199305534, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05448836833238602 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 30, + 22, + 18, + 25, + 21, + 29, + 13, + 18, + 26, + 22 + ], + "head_selection_time": "0:00:00.286959" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..f14d6cdeb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,457 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5154122867516343, + "mae": 0.4064919650554657, + "r2": 0.5020843148231506, + "score": -0.5154122867516343 + }, + "val": { + "rmse": 0.5559178796729317, + "mae": 0.4410634934902191, + "r2": 0.4172273278236389, + "score": -0.5559178796729317 + }, + "test": { + "rmse": 0.551186027318127, + "mae": 0.44573864340782166, + "r2": 0.4338840842247009, + "score": -0.551186027318127 + } + }, + "time": "0:07:03.426340", + "diversity": { + "train": { + "mean_centered_corr": -0.018950974568724632, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060403864830732346 + }, + "val": { + "mean_centered_corr": -0.019536394625902176, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06044872850179672 + }, + "test": { + "mean_centered_corr": -0.01978791505098343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06026185676455498 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 24, + 29, + 13, + 15, + 29, + 25, + 31, + 14, + 3 + ], + "head_selection_time": "0:00:00.257968" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..97ec96d27 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0/report.json @@ -0,0 +1,189 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5197141511819273, + "mae": 0.41487401723861694, + "r2": 0.49373799562454224, + "score": -0.5197141511819273 + }, + "val": { + "rmse": 0.5572907783524989, + "mae": 0.4426497519016266, + "r2": 0.41434532403945923, + "score": -0.5572907783524989 + }, + "test": { + "rmse": 0.5545303498210034, + "mae": 0.44824421405792236, + "r2": 0.42699337005615234, + "score": -0.5545303498210034 + } + }, + "time": "0:25:14.112066", + "diversity": { + "train": { + "mean_centered_corr": -0.020538095384836197, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056509166955947876 + }, + "val": { + "mean_centered_corr": -0.020886661484837532, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058363690972328186 + }, + "test": { + "mean_centered_corr": -0.020847400650382042, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057258859276771545 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.023366" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1/report.json new file mode 100644 index 000000000..5e5e30e0d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1/report.json @@ -0,0 +1,189 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5103319318333938, + "mae": 0.3996802568435669, + "r2": 0.5118517279624939, + "score": -0.5103319318333938 + }, + "val": { + "rmse": 0.5571610811723605, + "mae": 0.440170556306839, + "r2": 0.4146178960800171, + "score": -0.5571610811723605 + }, + "test": { + "rmse": 0.5529389828325415, + "mae": 0.4438725709915161, + "r2": 0.4302774667739868, + "score": -0.5529389828325415 + } + }, + "time": "0:26:36.215265", + "diversity": { + "train": { + "mean_centered_corr": -0.02035202831029892, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059233568608760834 + }, + "val": { + "mean_centered_corr": -0.020599549636244774, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06324069947004318 + }, + "test": { + "mean_centered_corr": -0.020422711968421936, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06101517006754875 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.023415" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2/report.json new file mode 100644 index 000000000..615f58ce6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2/report.json @@ -0,0 +1,189 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 819, + "metrics": { + "train": { + "rmse": 0.5218133353161428, + "mae": 0.4113169312477112, + "r2": 0.4896400570869446, + "score": -0.5218133353161428 + }, + "val": { + "rmse": 0.5577944659141506, + "mae": 0.4427873492240906, + "r2": 0.4132862091064453, + "score": -0.5577944659141506 + }, + "test": { + "rmse": 0.5536175199956386, + "mae": 0.4474765956401825, + "r2": 0.4288783073425293, + "score": -0.5536175199956386 + } + }, + "time": "0:22:50.050598", + "diversity": { + "train": { + "mean_centered_corr": -0.019786350429058075, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05782343074679375 + }, + "val": { + "mean_centered_corr": -0.018166765570640564, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05960576981306076 + }, + "test": { + "mean_centered_corr": -0.019035130739212036, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057923924177885056 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.023390" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml new file mode 100644 index 000000000..ce432034e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0/report.json new file mode 100644 index 000000000..f51c2c83d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0/report.json @@ -0,0 +1,185 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5146695445761407, + "mae": 0.40757593512535095, + "r2": 0.5035183429718018, + "score": -0.5146695445761407 + }, + "val": { + "rmse": 0.5559417888696784, + "mae": 0.44435763359069824, + "r2": 0.4171772003173828, + "score": -0.5559417888696784 + }, + "test": { + "rmse": 0.5541331785404355, + "mae": 0.44994547963142395, + "r2": 0.42781394720077515, + "score": -0.5541331785404355 + } + }, + "time": "0:25:14.112066", + "diversity": { + "train": { + "mean_centered_corr": -0.020538095384836197, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056509166955947876 + }, + "val": { + "mean_centered_corr": -0.020886661484837532, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058363690972328186 + }, + "test": { + "mean_centered_corr": -0.020847400650382042, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057258859276771545 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml new file mode 100644 index 000000000..2d5565099 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1/report.json new file mode 100644 index 000000000..d06153cf8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1/report.json @@ -0,0 +1,185 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5087316557555162, + "mae": 0.39884668588638306, + "r2": 0.5149083733558655, + "score": -0.5087316557555162 + }, + "val": { + "rmse": 0.5563776030607058, + "mae": 0.43999341130256653, + "r2": 0.4162631034851074, + "score": -0.5563776030607058 + }, + "test": { + "rmse": 0.5504132928776152, + "mae": 0.4417083263397217, + "r2": 0.4354702830314636, + "score": -0.5504132928776152 + } + }, + "time": "0:26:36.215265", + "diversity": { + "train": { + "mean_centered_corr": -0.02035202831029892, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059233568608760834 + }, + "val": { + "mean_centered_corr": -0.020599549636244774, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06324069947004318 + }, + "test": { + "mean_centered_corr": -0.020422711968421936, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06101517006754875 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml new file mode 100644 index 000000000..3610629d9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2/report.json new file mode 100644 index 000000000..0e8fd44b8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2/report.json @@ -0,0 +1,185 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 819, + "metrics": { + "train": { + "rmse": 0.518709062776439, + "mae": 0.407330721616745, + "r2": 0.49569422006607056, + "score": -0.518709062776439 + }, + "val": { + "rmse": 0.556831808887752, + "mae": 0.44260597229003906, + "r2": 0.4153096079826355, + "score": -0.556831808887752 + }, + "test": { + "rmse": 0.5516551774702309, + "mae": 0.4460088610649109, + "r2": 0.432919979095459, + "score": -0.5516551774702309 + } + }, + "time": "0:22:50.050598", + "diversity": { + "train": { + "mean_centered_corr": -0.019786350429058075, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05782343074679375 + }, + "val": { + "mean_centered_corr": -0.018166765570640564, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05960576981306076 + }, + "test": { + "mean_centered_corr": -0.019035130739212036, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057923924177885056 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5449bd7de --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,196 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5146023695303711, + "mae": 0.4073488712310791, + "r2": 0.5036479234695435, + "score": -0.5146023695303711 + }, + "val": { + "rmse": 0.5552988041877751, + "mae": 0.44173434376716614, + "r2": 0.4185245633125305, + "score": -0.5552988041877751 + }, + "test": { + "rmse": 0.5525187688548042, + "mae": 0.44692304730415344, + "r2": 0.43114304542541504, + "score": -0.5525187688548042 + } + }, + "time": "0:25:14.112066", + "diversity": { + "train": { + "mean_centered_corr": -0.020538095384836197, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056509166955947876 + }, + "val": { + "mean_centered_corr": -0.020886661484837532, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058363690972328186 + }, + "test": { + "mean_centered_corr": -0.020847400650382042, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057258859276771545 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 29, + 16, + 11, + 10, + 26, + 8, + 15, + 22 + ], + "head_selection_time": "0:00:00.230227" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..0ef3cfea2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1/report.json @@ -0,0 +1,193 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5090920101016853, + "mae": 0.39969104528427124, + "r2": 0.5142209529876709, + "score": -0.5090920101016853 + }, + "val": { + "rmse": 0.5558474595058753, + "mae": 0.44084590673446655, + "r2": 0.41737496852874756, + "score": -0.5558474595058753 + }, + "test": { + "rmse": 0.5521241448870569, + "mae": 0.44504550099372864, + "r2": 0.43195533752441406, + "score": -0.5521241448870569 + } + }, + "time": "0:26:36.215265", + "diversity": { + "train": { + "mean_centered_corr": -0.02035202831029892, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059233568608760834 + }, + "val": { + "mean_centered_corr": -0.020599549636244774, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06324069947004318 + }, + "test": { + "mean_centered_corr": -0.020422711968421936, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06101517006754875 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 22, + 29, + 30, + 4, + 13 + ], + "head_selection_time": "0:00:00.148296" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..4dadccf56 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 819, + "metrics": { + "train": { + "rmse": 0.5186568050088503, + "mae": 0.40700122714042664, + "r2": 0.4957958459854126, + "score": -0.5186568050088503 + }, + "val": { + "rmse": 0.5564955940706238, + "mae": 0.4424670338630676, + "r2": 0.41601550579071045, + "score": -0.5564955940706238 + }, + "test": { + "rmse": 0.5523659114897095, + "mae": 0.446776419878006, + "r2": 0.4314578175544739, + "score": -0.5523659114897095 + } + }, + "time": "0:22:50.050598", + "diversity": { + "train": { + "mean_centered_corr": -0.019786350429058075, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05782343074679375 + }, + "val": { + "mean_centered_corr": -0.018166765570640564, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05960576981306076 + }, + "test": { + "mean_centered_corr": -0.019035130739212036, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057923924177885056 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 31, + 6, + 25, + 8, + 19, + 22, + 2, + 12, + 18, + 29, + 5, + 13 + ], + "head_selection_time": "0:00:00.360643" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6e8396567 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5123514530013489, + "mae": 0.4078504741191864, + "r2": 0.5079806447029114, + "score": -0.5123514530013489 + }, + "val": { + "rmse": 0.5582863288518123, + "mae": 0.4448302686214447, + "r2": 0.41225099563598633, + "score": -0.5582863288518123 + }, + "test": { + "rmse": 0.5560390501031047, + "mae": 0.4517032206058502, + "r2": 0.4238712191581726, + "score": -0.5560390501031047 + } + }, + "time": "0:12:51.556071", + "diversity": { + "train": { + "mean_centered_corr": -0.02027146704494953, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06810352951288223 + }, + "val": { + "mean_centered_corr": -0.02014581859111786, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07032498717308044 + }, + "test": { + "mean_centered_corr": -0.02043480984866619, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0694202110171318 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023277" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..338753c41 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/1/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5159431112020112, + "mae": 0.4095075726509094, + "r2": 0.5010582208633423, + "score": -0.5159431112020112 + }, + "val": { + "rmse": 0.5579506166150647, + "mae": 0.4473210573196411, + "r2": 0.41295766830444336, + "score": -0.5579506166150647 + }, + "test": { + "rmse": 0.5600144253303337, + "mae": 0.4565167725086212, + "r2": 0.41560375690460205, + "score": -0.5600144253303337 + } + }, + "time": "0:11:50.725269", + "diversity": { + "train": { + "mean_centered_corr": -0.020498434081673622, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06282930076122284 + }, + "val": { + "mean_centered_corr": -0.020406439900398254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06703978031873703 + }, + "test": { + "mean_centered_corr": -0.020546942949295044, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06463686376810074 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.023511" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..6c097eae7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/2/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5152948652017916, + "mae": 0.40833914279937744, + "r2": 0.5023112297058105, + "score": -0.5152948652017916 + }, + "val": { + "rmse": 0.5578059530007539, + "mae": 0.4471565783023834, + "r2": 0.4132620096206665, + "score": -0.5578059530007539 + }, + "test": { + "rmse": 0.5592548752846349, + "mae": 0.45557308197021484, + "r2": 0.4171879291534424, + "score": -0.5592548752846349 + } + }, + "time": "0:11:47.829319", + "diversity": { + "train": { + "mean_centered_corr": -0.020578308030962944, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06090719997882843 + }, + "val": { + "mean_centered_corr": -0.020749453455209732, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06413886696100235 + }, + "test": { + "mean_centered_corr": -0.020869312807917595, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061883412301540375 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 28 + ], + "head_selection_time": "0:00:00.023299" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml new file mode 100644 index 000000000..4b93a10d8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0/report.json new file mode 100644 index 000000000..239380f66 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.504551673775848, + "mae": 0.39544814825057983, + "r2": 0.5228471159934998, + "score": -0.504551673775848 + }, + "val": { + "rmse": 0.5556360918188752, + "mae": 0.439733624458313, + "r2": 0.41781800985336304, + "score": -0.5556360918188752 + }, + "test": { + "rmse": 0.5494649592968895, + "mae": 0.4416595995426178, + "r2": 0.43741387128829956, + "score": -0.5494649592968895 + } + }, + "time": "0:12:51.556071", + "diversity": { + "train": { + "mean_centered_corr": -0.02027146704494953, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06810352951288223 + }, + "val": { + "mean_centered_corr": -0.02014581859111786, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07032498717308044 + }, + "test": { + "mean_centered_corr": -0.02043480984866619, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0694202110171318 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml new file mode 100644 index 000000000..41dae5a50 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1/report.json new file mode 100644 index 000000000..9ed689981 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5113957091590454, + "mae": 0.40314197540283203, + "r2": 0.5098145604133606, + "score": -0.5113957091590454 + }, + "val": { + "rmse": 0.5559996009180297, + "mae": 0.44189974665641785, + "r2": 0.41705596446990967, + "score": -0.5559996009180297 + }, + "test": { + "rmse": 0.5517844674752832, + "mae": 0.44495975971221924, + "r2": 0.43265408277511597, + "score": -0.5517844674752832 + } + }, + "time": "0:11:50.725269", + "diversity": { + "train": { + "mean_centered_corr": -0.020498434081673622, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06282930076122284 + }, + "val": { + "mean_centered_corr": -0.020406439900398254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06703978031873703 + }, + "test": { + "mean_centered_corr": -0.020546942949295044, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06463686376810074 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml new file mode 100644 index 000000000..76f4646c4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2/report.json new file mode 100644 index 000000000..13a846bc2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5112017866685475, + "mae": 0.402057021856308, + "r2": 0.5101861953735352, + "score": -0.5112017866685475 + }, + "val": { + "rmse": 0.555909194911898, + "mae": 0.44030460715293884, + "r2": 0.4172455668449402, + "score": -0.555909194911898 + }, + "test": { + "rmse": 0.5492662194956084, + "mae": 0.4420929551124573, + "r2": 0.43782079219818115, + "score": -0.5492662194956084 + } + }, + "time": "0:11:47.829319", + "diversity": { + "train": { + "mean_centered_corr": -0.020578308030962944, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06090719997882843 + }, + "val": { + "mean_centered_corr": -0.020749453455209732, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06413886696100235 + }, + "test": { + "mean_centered_corr": -0.020869312807917595, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061883412301540375 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..20d61ffa8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,190 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5047633843312425, + "mae": 0.3963008522987366, + "r2": 0.5224465727806091, + "score": -0.5047633843312425 + }, + "val": { + "rmse": 0.5551513290724005, + "mae": 0.4403177499771118, + "r2": 0.4188333749771118, + "score": -0.5551513290724005 + }, + "test": { + "rmse": 0.5507488105906536, + "mae": 0.44414353370666504, + "r2": 0.43478184938430786, + "score": -0.5507488105906536 + } + }, + "time": "0:12:51.556071", + "diversity": { + "train": { + "mean_centered_corr": -0.02027146704494953, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06810352951288223 + }, + "val": { + "mean_centered_corr": -0.02014581859111786, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07032498717308044 + }, + "test": { + "mean_centered_corr": -0.02043480984866619, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0694202110171318 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 21, + 10, + 6, + 29, + 12, + 17, + 15, + 8, + 22, + 16, + 21, + 4 + ], + "head_selection_time": "0:00:00.394226" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..19700c536 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5113803822251173, + "mae": 0.4032570719718933, + "r2": 0.5098439455032349, + "score": -0.5113803822251173 + }, + "val": { + "rmse": 0.5557057343261759, + "mae": 0.4411143362522125, + "r2": 0.4176720380783081, + "score": -0.5557057343261759 + }, + "test": { + "rmse": 0.5517152752978882, + "mae": 0.4449000060558319, + "r2": 0.4327963590621948, + "score": -0.5517152752978882 + } + }, + "time": "0:11:50.725269", + "diversity": { + "train": { + "mean_centered_corr": -0.020498434081673622, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06282930076122284 + }, + "val": { + "mean_centered_corr": -0.020406439900398254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06703978031873703 + }, + "test": { + "mean_centered_corr": -0.020546942949295044, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06463686376810074 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 22, + 20, + 29, + 28, + 1, + 24, + 30, + 10, + 19, + 14, + 26 + ], + "head_selection_time": "0:00:00.326139" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..d613ca389 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,187 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5123895805314972, + "mae": 0.4043997824192047, + "r2": 0.5079073905944824, + "score": -0.5123895805314972 + }, + "val": { + "rmse": 0.5554111756694609, + "mae": 0.4412407875061035, + "r2": 0.4182892441749573, + "score": -0.5554111756694609 + }, + "test": { + "rmse": 0.5509796867295551, + "mae": 0.4447599947452545, + "r2": 0.4343078136444092, + "score": -0.5509796867295551 + } + }, + "time": "0:11:47.829319", + "diversity": { + "train": { + "mean_centered_corr": -0.020578308030962944, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06090719997882843 + }, + "val": { + "mean_centered_corr": -0.020749453455209732, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06413886696100235 + }, + "test": { + "mean_centered_corr": -0.020869312807917595, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061883412301540375 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 28, + 11, + 31, + 14, + 3, + 0, + 13, + 31, + 27, + 16 + ], + "head_selection_time": "0:00:00.290963" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4d0b207d0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/0/report.json @@ -0,0 +1,204 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7046874761581421, + "min_features_per_member": 142.0, + "mean_features_per_member": 155.03125, + "max_features_per_member": 168.0, + "min_members_per_feature": 16.0, + "mean_members_per_feature": 22.549999237060547, + "max_members_per_feature": 29.0, + "mean_pairwise_jaccard": 0.5446517033602734, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5070207145078325, + "mae": 0.39629772305488586, + "r2": 0.5181657075881958, + "score": -0.5070207145078325 + }, + "val": { + "rmse": 0.5595885535124743, + "mae": 0.4446312487125397, + "r2": 0.4095059037208557, + "score": -0.5595885535124743 + }, + "test": { + "rmse": 0.5534224254724212, + "mae": 0.44561949372291565, + "r2": 0.42928075790405273, + "score": -0.5534224254724212 + } + }, + "time": "0:09:13.835446", + "diversity": { + "train": { + "mean_centered_corr": 0.022733226418495178, + "mean_pairwise_disagreement": NaN, + "member_std": 0.12898561358451843 + }, + "val": { + "mean_centered_corr": 0.03887920081615448, + "mean_pairwise_disagreement": NaN, + "member_std": 0.14495275914669037 + }, + "test": { + "mean_centered_corr": 0.04169675335288048, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1456397920846939 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.023598" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..d8bc9fda1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/1/report.json @@ -0,0 +1,204 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.7045454382896423, + "min_features_per_member": 142.0, + "mean_features_per_member": 155.0, + "max_features_per_member": 171.0, + "min_members_per_feature": 16.0, + "mean_members_per_feature": 22.545454025268555, + "max_members_per_feature": 29.0, + "mean_pairwise_jaccard": 0.5438212996120257, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5206705379376763, + "mae": 0.4162462055683136, + "r2": 0.49187296628952026, + "score": -0.5206705379376763 + }, + "val": { + "rmse": 0.5585395946793884, + "mae": 0.4492241144180298, + "r2": 0.41171765327453613, + "score": -0.5585395946793884 + }, + "test": { + "rmse": 0.5622948696369855, + "mae": 0.45980095863342285, + "r2": 0.4108346104621887, + "score": -0.5622948696369855 + } + }, + "time": "0:07:49.809972", + "diversity": { + "train": { + "mean_centered_corr": 0.0046985214576125145, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09994115680456161 + }, + "val": { + "mean_centered_corr": 0.01109667681157589, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11037178337574005 + }, + "test": { + "mean_centered_corr": 0.013022148050367832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1104893684387207 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 18 + ], + "head_selection_time": "0:00:00.023411" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..c031c7831 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/2/report.json @@ -0,0 +1,204 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.7042613625526428, + "min_features_per_member": 142.0, + "mean_features_per_member": 154.9375, + "max_features_per_member": 172.0, + "min_members_per_feature": 16.0, + "mean_members_per_feature": 22.53636360168457, + "max_members_per_feature": 29.0, + "mean_pairwise_jaccard": 0.5428619561922541, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3003, + "metrics": { + "train": { + "rmse": 0.5059006721989784, + "mae": 0.3979300558567047, + "r2": 0.5202922224998474, + "score": -0.5059006721989784 + }, + "val": { + "rmse": 0.5610582099606907, + "mae": 0.44673866033554077, + "r2": 0.406400203704834, + "score": -0.5610582099606907 + }, + "test": { + "rmse": 0.5524986222625401, + "mae": 0.4445948600769043, + "r2": 0.4311845302581787, + "score": -0.5524986222625401 + } + }, + "time": "0:09:37.350511", + "diversity": { + "train": { + "mean_centered_corr": 0.0061104861088097095, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11999407410621643 + }, + "val": { + "mean_centered_corr": 0.013669072650372982, + "mean_pairwise_disagreement": NaN, + "member_std": 0.13615167140960693 + }, + "test": { + "mean_centered_corr": 0.0167130958288908, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1356925070285797 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.023116" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml new file mode 100644 index 000000000..040e3d71f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.7 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..87e75b5e0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7046874761581421, + "min_features_per_member": 142.0, + "mean_features_per_member": 155.03125, + "max_features_per_member": 168.0, + "min_members_per_feature": 16.0, + "mean_members_per_feature": 22.549999237060547, + "max_members_per_feature": 29.0, + "mean_pairwise_jaccard": 0.5446517033602734, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5028727675143494, + "mae": 0.39573734998703003, + "r2": 0.5260173082351685, + "score": -0.5028727675143494 + }, + "val": { + "rmse": 0.5576335149669441, + "mae": 0.4440232217311859, + "r2": 0.41362476348876953, + "score": -0.5576335149669441 + }, + "test": { + "rmse": 0.5507854163911843, + "mae": 0.44484779238700867, + "r2": 0.4347066879272461, + "score": -0.5507854163911843 + } + }, + "time": "0:09:13.835446", + "diversity": { + "train": { + "mean_centered_corr": 0.022733226418495178, + "mean_pairwise_disagreement": NaN, + "member_std": 0.12898561358451843 + }, + "val": { + "mean_centered_corr": 0.03887920081615448, + "mean_pairwise_disagreement": NaN, + "member_std": 0.14495275914669037 + }, + "test": { + "mean_centered_corr": 0.04169675335288048, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1456397920846939 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml new file mode 100644 index 000000000..5dfdfdfed --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.7 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 1 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1/report.json new file mode 100644 index 000000000..f4a08e923 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.7045454382896423, + "min_features_per_member": 142.0, + "mean_features_per_member": 155.0, + "max_features_per_member": 171.0, + "min_members_per_feature": 16.0, + "mean_members_per_feature": 22.545454025268555, + "max_members_per_feature": 29.0, + "mean_pairwise_jaccard": 0.5438212996120257, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5134184759867957, + "mae": 0.40532490611076355, + "r2": 0.5059291124343872, + "score": -0.5134184759867957 + }, + "val": { + "rmse": 0.5557639999222178, + "mae": 0.4431096315383911, + "r2": 0.41754990816116333, + "score": -0.5557639999222178 + }, + "test": { + "rmse": 0.5537457867240355, + "mae": 0.44885125756263733, + "r2": 0.4286136031150818, + "score": -0.5537457867240355 + } + }, + "time": "0:07:49.809972", + "diversity": { + "train": { + "mean_centered_corr": 0.0046985214576125145, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09994115680456161 + }, + "val": { + "mean_centered_corr": 0.01109667681157589, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11037178337574005 + }, + "test": { + "mean_centered_corr": 0.013022148050367832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1104893684387207 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml new file mode 100644 index 000000000..87637db2a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.7 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 2 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2/report.json new file mode 100644 index 000000000..ab77cd273 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.7042613625526428, + "min_features_per_member": 142.0, + "mean_features_per_member": 154.9375, + "max_features_per_member": 172.0, + "min_members_per_feature": 16.0, + "mean_members_per_feature": 22.53636360168457, + "max_members_per_feature": 29.0, + "mean_pairwise_jaccard": 0.5428619561922541, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3003, + "metrics": { + "train": { + "rmse": 0.49698357865243403, + "mae": 0.39058470726013184, + "r2": 0.5370539426803589, + "score": -0.49698357865243403 + }, + "val": { + "rmse": 0.556207481645197, + "mae": 0.4430869519710541, + "r2": 0.4166199564933777, + "score": -0.556207481645197 + }, + "test": { + "rmse": 0.551272504610975, + "mae": 0.4455644190311432, + "r2": 0.4337064027786255, + "score": -0.551272504610975 + } + }, + "time": "0:09:37.350511", + "diversity": { + "train": { + "mean_centered_corr": 0.0061104861088097095, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11999407410621643 + }, + "val": { + "mean_centered_corr": 0.013669072650372982, + "mean_pairwise_disagreement": NaN, + "member_std": 0.13615167140960693 + }, + "test": { + "mean_centered_corr": 0.0167130958288908, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1356925070285797 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..22735cb5d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,211 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7046874761581421, + "min_features_per_member": 142.0, + "mean_features_per_member": 155.03125, + "max_features_per_member": 168.0, + "min_members_per_feature": 16.0, + "mean_members_per_feature": 22.549999237060547, + "max_members_per_feature": 29.0, + "mean_pairwise_jaccard": 0.5446517033602734, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.503753402357704, + "mae": 0.3957558870315552, + "r2": 0.5243557691574097, + "score": -0.503753402357704 + }, + "val": { + "rmse": 0.5551008108444078, + "mae": 0.43975356221199036, + "r2": 0.41893917322158813, + "score": -0.5551008108444078 + }, + "test": { + "rmse": 0.5501919833980724, + "mae": 0.44342172145843506, + "r2": 0.4359241724014282, + "score": -0.5501919833980724 + } + }, + "time": "0:09:13.835446", + "diversity": { + "train": { + "mean_centered_corr": 0.022733226418495178, + "mean_pairwise_disagreement": NaN, + "member_std": 0.12898561358451843 + }, + "val": { + "mean_centered_corr": 0.03887920081615448, + "mean_pairwise_disagreement": NaN, + "member_std": 0.14495275914669037 + }, + "test": { + "mean_centered_corr": 0.04169675335288048, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1456397920846939 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 8, + 11, + 4, + 16, + 2, + 7, + 23, + 18 + ], + "head_selection_time": "0:00:00.227173" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..e6e14c6ab --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,211 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.7045454382896423, + "min_features_per_member": 142.0, + "mean_features_per_member": 155.0, + "max_features_per_member": 171.0, + "min_members_per_feature": 16.0, + "mean_members_per_feature": 22.545454025268555, + "max_members_per_feature": 29.0, + "mean_pairwise_jaccard": 0.5438212996120257, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5131697140167799, + "mae": 0.40394970774650574, + "r2": 0.5064077377319336, + "score": -0.5131697140167799 + }, + "val": { + "rmse": 0.5549583042651809, + "mae": 0.4405714273452759, + "r2": 0.41923749446868896, + "score": -0.5549583042651809 + }, + "test": { + "rmse": 0.5533328640985606, + "mae": 0.4472924470901489, + "r2": 0.42946547269821167, + "score": -0.5533328640985606 + } + }, + "time": "0:07:49.809972", + "diversity": { + "train": { + "mean_centered_corr": 0.0046985214576125145, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09994115680456161 + }, + "val": { + "mean_centered_corr": 0.01109667681157589, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11037178337574005 + }, + "test": { + "mean_centered_corr": 0.013022148050367832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1104893684387207 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 18, + 11, + 22, + 24, + 4, + 12, + 30, + 9 + ], + "head_selection_time": "0:00:00.233733" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..08f856a37 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,220 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.7042613625526428, + "min_features_per_member": 142.0, + "mean_features_per_member": 154.9375, + "max_features_per_member": 172.0, + "min_members_per_feature": 16.0, + "mean_members_per_feature": 22.53636360168457, + "max_members_per_feature": 29.0, + "mean_pairwise_jaccard": 0.5428619561922541, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3003, + "metrics": { + "train": { + "rmse": 0.4972497582432623, + "mae": 0.3897043466567993, + "r2": 0.5365579128265381, + "score": -0.4972497582432623 + }, + "val": { + "rmse": 0.55517701590878, + "mae": 0.4405524730682373, + "r2": 0.4187796115875244, + "score": -0.55517701590878 + }, + "test": { + "rmse": 0.5505736480564649, + "mae": 0.4438551664352417, + "r2": 0.43514126539230347, + "score": -0.5505736480564649 + } + }, + "time": "0:09:37.350511", + "diversity": { + "train": { + "mean_centered_corr": 0.0061104861088097095, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11999407410621643 + }, + "val": { + "mean_centered_corr": 0.013669072650372982, + "mean_pairwise_disagreement": NaN, + "member_std": 0.13615167140960693 + }, + "test": { + "mean_centered_corr": 0.0167130958288908, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1356925070285797 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 0, + 23, + 13, + 7, + 5, + 11, + 8, + 30, + 14, + 26, + 3, + 4, + 13, + 28, + 5, + 7, + 23 + ], + "head_selection_time": "0:00:00.548122" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c496cfa72 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5169435461643365, + "mae": 0.4078221619129181, + "r2": 0.49912136793136597, + "score": -0.5169435461643365 + }, + "val": { + "rmse": 0.5574634025946379, + "mae": 0.44402608275413513, + "r2": 0.41398245096206665, + "score": -0.5574634025946379 + }, + "test": { + "rmse": 0.5547066537478277, + "mae": 0.450179785490036, + "r2": 0.4266290068626404, + "score": -0.5547066537478277 + } + }, + "time": "0:09:33.111358", + "diversity": { + "train": { + "mean_centered_corr": -0.020444847643375397, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059093035757541656 + }, + "val": { + "mean_centered_corr": -0.020575184375047684, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06087604537606239 + }, + "test": { + "mean_centered_corr": -0.020676201209425926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060276810079813004 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 18 + ], + "head_selection_time": "0:00:00.023373" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..de4494e93 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.516247633496788, + "mae": 0.40711885690689087, + "r2": 0.5004690289497375, + "score": -0.516247633496788 + }, + "val": { + "rmse": 0.5576988200546102, + "mae": 0.4442465603351593, + "r2": 0.41348737478256226, + "score": -0.5576988200546102 + }, + "test": { + "rmse": 0.551726996987587, + "mae": 0.44564470648765564, + "r2": 0.43277227878570557, + "score": -0.551726996987587 + } + }, + "time": "0:09:34.536942", + "diversity": { + "train": { + "mean_centered_corr": -0.020366203039884567, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05432252958416939 + }, + "val": { + "mean_centered_corr": -0.020544419065117836, + "mean_pairwise_disagreement": NaN, + "member_std": 0.054353535175323486 + }, + "test": { + "mean_centered_corr": -0.02033921517431736, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05365084111690521 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 5 + ], + "head_selection_time": "0:00:00.023071" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..3123c6e12 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5148993790576664, + "mae": 0.40408939123153687, + "r2": 0.5030747652053833, + "score": -0.5148993790576664 + }, + "val": { + "rmse": 0.5580165521873937, + "mae": 0.4407314658164978, + "r2": 0.41281890869140625, + "score": -0.5580165521873937 + }, + "test": { + "rmse": 0.5499792279309027, + "mae": 0.44245660305023193, + "r2": 0.43636035919189453, + "score": -0.5499792279309027 + } + }, + "time": "0:09:31.116943", + "diversity": { + "train": { + "mean_centered_corr": -0.020301729440689087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05993633717298508 + }, + "val": { + "mean_centered_corr": -0.020580198615789413, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06198854371905327 + }, + "test": { + "mean_centered_corr": -0.02039889432489872, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05979244410991669 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023734" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml new file mode 100644 index 000000000..6d6f2bc2b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0/report.json new file mode 100644 index 000000000..03a93045c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5137012893899691, + "mae": 0.40439915657043457, + "r2": 0.5053846836090088, + "score": -0.5137012893899691 + }, + "val": { + "rmse": 0.5556336245387727, + "mae": 0.4420306384563446, + "r2": 0.4178231954574585, + "score": -0.5556336245387727 + }, + "test": { + "rmse": 0.5518591323593222, + "mae": 0.4463585913181305, + "r2": 0.43250054121017456, + "score": -0.5518591323593222 + } + }, + "time": "0:09:33.111358", + "diversity": { + "train": { + "mean_centered_corr": -0.020444847643375397, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059093035757541656 + }, + "val": { + "mean_centered_corr": -0.020575184375047684, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06087604537606239 + }, + "test": { + "mean_centered_corr": -0.020676201209425926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060276810079813004 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml new file mode 100644 index 000000000..d02cc3fe9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1/report.json new file mode 100644 index 000000000..b0272ba3f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5138557888087629, + "mae": 0.4044031500816345, + "r2": 0.50508713722229, + "score": -0.5138557888087629 + }, + "val": { + "rmse": 0.5562834011722252, + "mae": 0.44168752431869507, + "r2": 0.41646069288253784, + "score": -0.5562834011722252 + }, + "test": { + "rmse": 0.5505008118837418, + "mae": 0.44447061419487, + "r2": 0.43529075384140015, + "score": -0.5505008118837418 + } + }, + "time": "0:09:34.536942", + "diversity": { + "train": { + "mean_centered_corr": -0.020366203039884567, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05432252958416939 + }, + "val": { + "mean_centered_corr": -0.020544419065117836, + "mean_pairwise_disagreement": NaN, + "member_std": 0.054353535175323486 + }, + "test": { + "mean_centered_corr": -0.02033921517431736, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05365084111690521 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml new file mode 100644 index 000000000..6e4fdc141 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2/report.json new file mode 100644 index 000000000..2cde9fc4f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5124919088236671, + "mae": 0.40055063366889954, + "r2": 0.5077108144760132, + "score": -0.5124919088236671 + }, + "val": { + "rmse": 0.5564119371157284, + "mae": 0.4400753974914551, + "r2": 0.416191041469574, + "score": -0.5564119371157284 + }, + "test": { + "rmse": 0.5491394300416609, + "mae": 0.44087520241737366, + "r2": 0.4380803108215332, + "score": -0.5491394300416609 + } + }, + "time": "0:09:31.116943", + "diversity": { + "train": { + "mean_centered_corr": -0.020301729440689087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05993633717298508 + }, + "val": { + "mean_centered_corr": -0.020580198615789413, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06198854371905327 + }, + "test": { + "mean_centered_corr": -0.02039889432489872, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05979244410991669 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..882c6c418 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.514097726188904, + "mae": 0.4052426517009735, + "r2": 0.5046209692955017, + "score": -0.514097726188904 + }, + "val": { + "rmse": 0.5552720495549479, + "mae": 0.441690057516098, + "r2": 0.4185805916786194, + "score": -0.5552720495549479 + }, + "test": { + "rmse": 0.5524877260752137, + "mae": 0.447174072265625, + "r2": 0.43120694160461426, + "score": -0.5524877260752137 + } + }, + "time": "0:09:33.111358", + "diversity": { + "train": { + "mean_centered_corr": -0.020444847643375397, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059093035757541656 + }, + "val": { + "mean_centered_corr": -0.020575184375047684, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06087604537606239 + }, + "test": { + "mean_centered_corr": -0.020676201209425926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060276810079813004 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 18, + 10, + 11, + 26, + 21, + 14, + 31, + 19, + 11, + 16, + 26, + 10, + 11, + 15, + 21, + 29 + ], + "head_selection_time": "0:00:00.514504" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..68f3842ad --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,187 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5148582248071162, + "mae": 0.40721189975738525, + "r2": 0.5031542778015137, + "score": -0.5148582248071162 + }, + "val": { + "rmse": 0.55584140086384, + "mae": 0.4416872560977936, + "r2": 0.4173876643180847, + "score": -0.55584140086384 + }, + "test": { + "rmse": 0.5514378793216401, + "mae": 0.44598153233528137, + "r2": 0.433366596698761, + "score": -0.5514378793216401 + } + }, + "time": "0:09:34.536942", + "diversity": { + "train": { + "mean_centered_corr": -0.020366203039884567, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05432252958416939 + }, + "val": { + "mean_centered_corr": -0.020544419065117836, + "mean_pairwise_disagreement": NaN, + "member_std": 0.054353535175323486 + }, + "test": { + "mean_centered_corr": -0.02033921517431736, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05365084111690521 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 5, + 24, + 30, + 29, + 25, + 21, + 1, + 31, + 9 + ], + "head_selection_time": "0:00:00.253830" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..ff59a7559 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,189 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5130983639400511, + "mae": 0.40277671813964844, + "r2": 0.5065450072288513, + "score": -0.5130983639400511 + }, + "val": { + "rmse": 0.555843679564389, + "mae": 0.4404619336128235, + "r2": 0.4173828959465027, + "score": -0.555843679564389 + }, + "test": { + "rmse": 0.5502940789713319, + "mae": 0.4434540569782257, + "r2": 0.4357147812843323, + "score": -0.5502940789713319 + } + }, + "time": "0:09:31.116943", + "diversity": { + "train": { + "mean_centered_corr": -0.020301729440689087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05993633717298508 + }, + "val": { + "mean_centered_corr": -0.020580198615789413, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06198854371905327 + }, + "test": { + "mean_centered_corr": -0.02039889432489872, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05979244410991669 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 4, + 3, + 18, + 0, + 14, + 25, + 29, + 22, + 6, + 0 + ], + "head_selection_time": "0:00:00.326608" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0da12dd14 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7835229427379597, + "recall": 0.9817888071846897, + "f1-score": 0.871522047464133, + "support": 84179.0 + }, + "1": { + "precision": 0.602952602952603, + "recall": 0.09252046737143312, + "f1-score": 0.1604244909209937, + "support": 25162.0 + }, + "accuracy": 0.7771467244674916, + "macro avg": { + "precision": 0.6932377728452814, + "recall": 0.5371546372780615, + "f1-score": 0.5159732691925634, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7419693545168977, + "recall": 0.7771467244674916, + "f1-score": 0.707881357167369, + "support": 109341.0 + }, + "cross-entropy": 0.49822654924283316, + "roc-auc": 0.6867900811541505, + "score": 0.6867900811541505 + }, + "val": { + "0": { + "precision": 0.6877843221759325, + "recall": 0.9255703054517335, + "f1-score": 0.7891541441169199, + "support": 15518.0 + }, + "1": { + "precision": 0.6580817051509769, + "recall": 0.2542605512981814, + "f1-score": 0.36680141902483293, + "support": 8743.0 + }, + "accuracy": 0.6836486542187049, + "macro avg": { + "precision": 0.6729330136634547, + "recall": 0.5899154283749575, + "f1-score": 0.5779777815708764, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6770803124216278, + "recall": 0.6836486542187049, + "f1-score": 0.6369497883409784, + "support": 24261.0 + }, + "cross-entropy": 0.6337455659020594, + "roc-auc": 0.6404461738890127, + "score": 0.6404461738890127 + }, + "test": { + "0": { + "precision": 0.6663787430075683, + "recall": 0.9573927431745657, + "f1-score": 0.785807828491051, + "support": 16922.0 + }, + "1": { + "precision": 0.6635557629491368, + "recall": 0.14916605475715933, + "f1-score": 0.24357656731757452, + "support": 9533.0 + }, + "accuracy": 0.6661500661500661, + "macro avg": { + "precision": 0.6649672529783526, + "recall": 0.5532793989658625, + "f1-score": 0.5146921979043128, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6653614884660062, + "recall": 0.6661500661500661, + "f1-score": 0.5904160079366473, + "support": 26455.0 + }, + "cross-entropy": 0.6685406788276543, + "roc-auc": 0.6034781946000056, + "score": 0.6034781946000056 + } + }, + "time": "0:01:00.788883", + "diversity": { + "train": { + "mean_centered_corr": -0.013087093830108643, + "mean_pairwise_disagreement": 0.025822423398123243, + "member_std": 0.02709772437810898 + }, + "val": { + "mean_centered_corr": -0.004288136959075928, + "mean_pairwise_disagreement": 0.08692423855623854, + "member_std": 0.060995232313871384 + }, + "test": { + "mean_centered_corr": 0.0036114430986344814, + "mean_pairwise_disagreement": 0.05469726437468373, + "member_std": 0.05942297354340553 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 14 + ], + "head_selection_time": "0:00:04.762789" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1/report.json new file mode 100644 index 000000000..baebe29e5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7921504831670823, + "recall": 0.9660247805272099, + "f1-score": 0.8704900045494688, + "support": 84179.0 + }, + "1": { + "precision": 0.5721765145848915, + "recall": 0.1520149431682696, + "f1-score": 0.2402110088862373, + "support": 25162.0 + }, + "accuracy": 0.7787014934928344, + "macro avg": { + "precision": 0.6821634988759868, + "recall": 0.5590198618477398, + "f1-score": 0.555350506717853, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7415291700506383, + "recall": 0.7787014934928344, + "f1-score": 0.7254476134164242, + "support": 109341.0 + }, + "cross-entropy": 0.4932425445384841, + "roc-auc": 0.6956628076755741, + "score": 0.6956628076755741 + }, + "val": { + "0": { + "precision": 0.6824526887764661, + "recall": 0.9388452120118572, + "f1-score": 0.7903759561655727, + "support": 15518.0 + }, + "1": { + "precision": 0.6742190181943014, + "recall": 0.22463685233901406, + "f1-score": 0.3369938229238161, + "support": 8743.0 + }, + "accuracy": 0.6814640781501174, + "macro avg": { + "precision": 0.6783358534853838, + "recall": 0.5817410321754356, + "f1-score": 0.5636848895446944, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6794854993819701, + "recall": 0.6814640781501174, + "f1-score": 0.6269894514488389, + "support": 24261.0 + }, + "cross-entropy": 0.6629784158321528, + "roc-auc": 0.6313621036574809, + "score": 0.6313621036574809 + }, + "test": { + "0": { + "precision": 0.6617878311358413, + "recall": 0.9699208131426545, + "f1-score": 0.7867603000742995, + "support": 16922.0 + }, + "1": { + "precision": 0.6922611850060459, + "recall": 0.12010909472359173, + "f1-score": 0.2047018861178153, + "support": 9533.0 + }, + "accuracy": 0.6636930636930637, + "macro avg": { + "precision": 0.6770245080709436, + "recall": 0.5450149539331232, + "f1-score": 0.49573109309605745, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6727688359532542, + "recall": 0.6636930636930637, + "f1-score": 0.5770168542135109, + "support": 26455.0 + }, + "cross-entropy": 0.7192501404441878, + "roc-auc": 0.5963646574673217, + "score": 0.5963646574673217 + } + }, + "time": "0:01:04.856616", + "diversity": { + "train": { + "mean_centered_corr": -0.009633250534534454, + "mean_pairwise_disagreement": 0.02490320677749485, + "member_std": 0.024462362751364708 + }, + "val": { + "mean_centered_corr": 0.023074615746736526, + "mean_pairwise_disagreement": 0.06982341565049974, + "member_std": 0.0468476265668869 + }, + "test": { + "mean_centered_corr": 0.04183991998434067, + "mean_pairwise_disagreement": 0.04013975344620506, + "member_std": 0.043004557490348816 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 15 + ], + "head_selection_time": "0:00:04.849572" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2/report.json new file mode 100644 index 000000000..0f03817b9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7892154495494365, + "recall": 0.9707052827902446, + "f1-score": 0.87060239936926, + "support": 84179.0 + }, + "1": { + "precision": 0.5751206064782909, + "recall": 0.13266036086161673, + "f1-score": 0.215591293676936, + "support": 25162.0 + }, + "accuracy": 0.7778509433789704, + "macro avg": { + "precision": 0.6821680280138637, + "recall": 0.5516828218259306, + "f1-score": 0.543096846523098, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7399470649420506, + "recall": 0.7778509433789704, + "f1-score": 0.7198685534978095, + "support": 109341.0 + }, + "cross-entropy": 0.49882900198018437, + "roc-auc": 0.6810205996481966, + "score": 0.6810205996481966 + }, + "val": { + "0": { + "precision": 0.6962851665716803, + "recall": 0.8660265498131202, + "f1-score": 0.7719348631494299, + "support": 15518.0 + }, + "1": { + "precision": 0.5808467741935484, + "recall": 0.32952075946471465, + "f1-score": 0.4204918630956725, + "support": 8743.0 + }, + "accuracy": 0.6726845554593792, + "macro avg": { + "precision": 0.6385659703826143, + "recall": 0.5977736546389174, + "f1-score": 0.5962133631225512, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6546843312985255, + "recall": 0.6726845554593792, + "f1-score": 0.6452844303779035, + "support": 24261.0 + }, + "cross-entropy": 0.6307411915692648, + "roc-auc": 0.6342271136151092, + "score": 0.6342271136151092 + }, + "test": { + "0": { + "precision": 0.670945256715403, + "recall": 0.932868455265335, + "f1-score": 0.7805191594561187, + "support": 16922.0 + }, + "1": { + "precision": 0.6118893064571234, + "recall": 0.18787370187768804, + "f1-score": 0.28747993579454256, + "support": 9533.0 + }, + "accuracy": 0.6644112644112644, + "macro avg": { + "precision": 0.6414172815862631, + "recall": 0.5603710785715115, + "f1-score": 0.5339995476253306, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6496645773046232, + "recall": 0.6644112644112644, + "f1-score": 0.6028535794460712, + "support": 26455.0 + }, + "cross-entropy": 0.6521463035772068, + "roc-auc": 0.5992387053088735, + "score": 0.5992387053088735 + } + }, + "time": "0:01:00.829886", + "diversity": { + "train": { + "mean_centered_corr": -0.0152942705899477, + "mean_pairwise_disagreement": 0.023205388676030093, + "member_std": 0.024158649146556854 + }, + "val": { + "mean_centered_corr": -0.00040229607839137316, + "mean_pairwise_disagreement": 0.06998355252223468, + "member_std": 0.05039098858833313 + }, + "test": { + "mean_centered_corr": 0.0061957864090800285, + "mean_pairwise_disagreement": 0.04117994037348876, + "member_std": 0.047571223229169846 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:04.818158" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..bf2cd1131 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..9b9138da9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/0/report.json @@ -0,0 +1,215 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7862557522292872, + "recall": 0.9783200085532021, + "f1-score": 0.8718353174078054, + "support": 84179.0 + }, + "1": { + "precision": 0.6031746031746031, + "recall": 0.11024560845719736, + "f1-score": 0.18641846712140048, + "support": 25162.0 + }, + "accuracy": 0.7785551622904492, + "macro avg": { + "precision": 0.6947151777019451, + "recall": 0.5442828085051997, + "f1-score": 0.529126892264603, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441243662668947, + "recall": 0.7785551622904492, + "f1-score": 0.7141043767093801, + "support": 109341.0 + }, + "cross-entropy": 0.4922807370219411, + "roc-auc": 0.6989988975549913, + "score": 0.6989988975549913 + }, + "val": { + "0": { + "precision": 0.6750500272876114, + "recall": 0.9565021265627014, + "f1-score": 0.7915000266624007, + "support": 15518.0 + }, + "1": { + "precision": 0.7030356357237132, + "recall": 0.1827747912615807, + "f1-score": 0.29012345679012347, + "support": 8743.0 + }, + "accuracy": 0.6776719838423808, + "macro avg": { + "precision": 0.6890428315056623, + "recall": 0.569638458912141, + "f1-score": 0.5408117417262621, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6851352741676592, + "recall": 0.6776719838423808, + "f1-score": 0.6108176413364323, + "support": 24261.0 + }, + "cross-entropy": 0.6872792555776136, + "roc-auc": 0.6227054517511603, + "score": 0.6227054517511603 + }, + "test": { + "0": { + "precision": 0.656363924552227, + "recall": 0.9788441082614349, + "f1-score": 0.7858057782627259, + "support": 16922.0 + }, + "1": { + "precision": 0.7063166529942576, + "recall": 0.09031784328123361, + "f1-score": 0.16015625, + "support": 9533.0 + }, + "accuracy": 0.6586656586656586, + "macro avg": { + "precision": 0.6813402887732423, + "recall": 0.5345809757713342, + "f1-score": 0.47298101413136295, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6743642783695727, + "recall": 0.6586656586656586, + "f1-score": 0.5603543719906198, + "support": 26455.0 + }, + "cross-entropy": 0.756433532699498, + "roc-auc": 0.591151981311678, + "score": 0.591151981311678 + } + }, + "time": "0:01:00.788883", + "diversity": { + "train": { + "mean_centered_corr": -0.013087093830108643, + "mean_pairwise_disagreement": 0.025822423398123243, + "member_std": 0.02709772437810898 + }, + "val": { + "mean_centered_corr": -0.004288136959075928, + "mean_pairwise_disagreement": 0.08692423855623854, + "member_std": 0.060995232313871384 + }, + "test": { + "mean_centered_corr": 0.0036114430986344814, + "mean_pairwise_disagreement": 0.05469726437468373, + "member_std": 0.05942297354340553 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml new file mode 100644 index 000000000..48a2a15be --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/1/report.json new file mode 100644 index 000000000..e74ec9d76 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/1/report.json @@ -0,0 +1,215 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7880169294701383, + "recall": 0.9754095439480155, + "f1-score": 0.8717564870259481, + "support": 84179.0 + }, + "1": { + "precision": 0.5975894245723172, + "recall": 0.12216834909784596, + "f1-score": 0.202864119316307, + "support": 25162.0 + }, + "accuracy": 0.7790581757986482, + "macro avg": { + "precision": 0.6928031770212277, + "recall": 0.5487889465229308, + "f1-score": 0.5373103031711275, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441949699285303, + "recall": 0.7790581757986482, + "f1-score": 0.7178282281266332, + "support": 109341.0 + }, + "cross-entropy": 0.490351542028749, + "roc-auc": 0.7032208565488707, + "score": 0.7032208565488707 + }, + "val": { + "0": { + "precision": 0.6735332819093425, + "recall": 0.9565665678566826, + "f1-score": 0.7904784727214633, + "support": 15518.0 + }, + "1": { + "precision": 0.6966696669666966, + "recall": 0.17705593045865264, + "f1-score": 0.2823529411764706, + "support": 8743.0 + }, + "accuracy": 0.6756522814393471, + "macro avg": { + "precision": 0.6851014744380195, + "recall": 0.5668112491576676, + "f1-score": 0.536415706948967, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6818710014821732, + "recall": 0.6756522814393471, + "f1-score": 0.6073639464324451, + "support": 24261.0 + }, + "cross-entropy": 0.713781615665886, + "roc-auc": 0.6203331748306973, + "score": 0.6203331748306973 + }, + "test": { + "0": { + "precision": 0.6556079021338929, + "recall": 0.9786077295827916, + "f1-score": 0.7851876437258482, + "support": 16922.0 + }, + "1": { + "precision": 0.697324414715719, + "recall": 0.0874855764187559, + "f1-score": 0.1554664926833815, + "support": 9533.0 + }, + "accuracy": 0.6574938574938575, + "macro avg": { + "precision": 0.676466158424806, + "recall": 0.5330466530007738, + "f1-score": 0.47032706820461484, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6706403540122731, + "recall": 0.6574938574938575, + "f1-score": 0.5582690373040816, + "support": 26455.0 + }, + "cross-entropy": 0.7978280002692709, + "roc-auc": 0.5897210540664094, + "score": 0.5897210540664094 + } + }, + "time": "0:01:04.856616", + "diversity": { + "train": { + "mean_centered_corr": -0.009633250534534454, + "mean_pairwise_disagreement": 0.02490320677749485, + "member_std": 0.024462362751364708 + }, + "val": { + "mean_centered_corr": 0.023074615746736526, + "mean_pairwise_disagreement": 0.06982341565049974, + "member_std": 0.0468476265668869 + }, + "test": { + "mean_centered_corr": 0.04183991998434067, + "mean_pairwise_disagreement": 0.04013975344620506, + "member_std": 0.043004557490348816 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml new file mode 100644 index 000000000..ab67f52a6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/2/report.json new file mode 100644 index 000000000..0f9aca3a7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-evaluation/2/report.json @@ -0,0 +1,215 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.786010743352193, + "recall": 0.978640753632141, + "f1-score": 0.8718119668973692, + "support": 84179.0 + }, + "1": { + "precision": 0.6032656663724625, + "recall": 0.10865590970511088, + "f1-score": 0.18414494510675558, + "support": 25162.0 + }, + "accuracy": 0.7784362681885112, + "macro avg": { + "precision": 0.6946382048623277, + "recall": 0.5436483316686259, + "f1-score": 0.5279784560020624, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439566956759877, + "recall": 0.7784362681885112, + "f1-score": 0.7135632074906011, + "support": 109341.0 + }, + "cross-entropy": 0.4919100581485962, + "roc-auc": 0.6999783629949486, + "score": 0.6999783629949486 + }, + "val": { + "0": { + "precision": 0.676180922852453, + "recall": 0.955664389740946, + "f1-score": 0.7919893190921228, + "support": 15518.0 + }, + "1": { + "precision": 0.7045942464577072, + "recall": 0.18769301155209883, + "f1-score": 0.2964234104046243, + "support": 8743.0 + }, + "accuracy": 0.678908536334034, + "macro avg": { + "precision": 0.6903875846550801, + "recall": 0.5716787006465225, + "f1-score": 0.5442063647483735, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6864203065662627, + "recall": 0.678908536334034, + "f1-score": 0.6134009369291947, + "support": 24261.0 + }, + "cross-entropy": 0.7026160031895037, + "roc-auc": 0.6212503263524413, + "score": 0.6212503263524413 + }, + "test": { + "0": { + "precision": 0.6575974542561654, + "recall": 0.9769530788322893, + "f1-score": 0.7860776948314393, + "support": 16922.0 + }, + "1": { + "precision": 0.7034220532319392, + "recall": 0.09703136473303263, + "f1-score": 0.17053834808259588, + "support": 9533.0 + }, + "accuracy": 0.6598752598752599, + "macro avg": { + "precision": 0.6805097537440523, + "recall": 0.5369922217826609, + "f1-score": 0.47830802145701756, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6741102458659197, + "recall": 0.6598752598752599, + "f1-score": 0.5642694698245702, + "support": 26455.0 + }, + "cross-entropy": 0.780454852534039, + "roc-auc": 0.5893011087345269, + "score": 0.5893011087345269 + } + }, + "time": "0:01:00.829886", + "diversity": { + "train": { + "mean_centered_corr": -0.0152942705899477, + "mean_pairwise_disagreement": 0.023205388676030093, + "member_std": 0.024158649146556854 + }, + "val": { + "mean_centered_corr": -0.00040229607839137316, + "mean_pairwise_disagreement": 0.06998355252223468, + "member_std": 0.05039098858833313 + }, + "test": { + "mean_centered_corr": 0.0061957864090800285, + "mean_pairwise_disagreement": 0.04117994037348876, + "member_std": 0.047571223229169846 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5f26c2628 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,227 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7829606930730526, + "recall": 0.9834281709214887, + "f1-score": 0.8718189046448073, + "support": 84179.0 + }, + "1": { + "precision": 0.6134663341645885, + "recall": 0.08798982592798664, + "f1-score": 0.15390497375829829, + "support": 25162.0 + }, + "accuracy": 0.7773662212710694, + "macro avg": { + "precision": 0.6982135136188206, + "recall": 0.5357089984247376, + "f1-score": 0.5128619392015528, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439559550621073, + "recall": 0.7773662212710694, + "f1-score": 0.7066096022882681, + "support": 109341.0 + }, + "cross-entropy": 0.4977984361309486, + "roc-auc": 0.6854164410903827, + "score": 0.6854164410903827 + }, + "val": { + "0": { + "precision": 0.6785484611851171, + "recall": 0.9519267946900374, + "f1-score": 0.7923192447972538, + "support": 15518.0 + }, + "1": { + "precision": 0.7005218787635488, + "recall": 0.19958824202218917, + "f1-score": 0.31066405554566495, + "support": 8743.0 + }, + "accuracy": 0.6808045834879024, + "macro avg": { + "precision": 0.689535169974333, + "recall": 0.5757575183561132, + "f1-score": 0.5514916501714594, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6864670791270086, + "recall": 0.6808045834879024, + "f1-score": 0.6187439049668, + "support": 24261.0 + }, + "cross-entropy": 0.6384330519984234, + "roc-auc": 0.6413971749638401, + "score": 0.6413971749638401 + }, + "test": { + "0": { + "precision": 0.659637469489016, + "recall": 0.9741756293582319, + "f1-score": 0.7866294467110443, + "support": 16922.0 + }, + "1": { + "precision": 0.7015027322404371, + "recall": 0.1077310395468373, + "f1-score": 0.18677821223970173, + "support": 9533.0 + }, + "accuracy": 0.661954261954262, + "macro avg": { + "precision": 0.6805701008647266, + "recall": 0.5409533344525346, + "f1-score": 0.486703829475373, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6747235231578611, + "recall": 0.661954261954262, + "f1-score": 0.5704743978274568, + "support": 26455.0 + }, + "cross-entropy": 0.6791357747346614, + "roc-auc": 0.6040305589800323, + "score": 0.6040305589800323 + } + }, + "time": "0:01:00.788883", + "diversity": { + "train": { + "mean_centered_corr": -0.013087093830108643, + "mean_pairwise_disagreement": 0.025822423398123243, + "member_std": 0.02709772437810898 + }, + "val": { + "mean_centered_corr": -0.004288136959075928, + "mean_pairwise_disagreement": 0.08692423855623854, + "member_std": 0.060995232313871384 + }, + "test": { + "mean_centered_corr": 0.0036114430986344814, + "mean_pairwise_disagreement": 0.05469726437468373, + "member_std": 0.05942297354340553 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 14, + 0, + 25, + 25, + 0, + 25, + 0, + 25, + 0 + ], + "head_selection_time": "0:00:48.737044" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..376991fac --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1/report.json @@ -0,0 +1,223 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7932819825198438, + "recall": 0.9628529680799249, + "f1-score": 0.8698806559629089, + "support": 84179.0 + }, + "1": { + "precision": 0.5637555803571429, + "recall": 0.1605993164295366, + "f1-score": 0.24998453448809155, + "support": 25162.0 + }, + "accuracy": 0.7782350627852315, + "macro avg": { + "precision": 0.6785187814384934, + "recall": 0.5617261422547307, + "f1-score": 0.5599325952255002, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7404624241545658, + "recall": 0.7782350627852315, + "f1-score": 0.7272276144821345, + "support": 109341.0 + }, + "cross-entropy": 0.4937799085537247, + "roc-auc": 0.6959439172205661, + "score": 0.6959439172205661 + }, + "val": { + "0": { + "precision": 0.6909425020552251, + "recall": 0.9207372084031448, + "f1-score": 0.7894576898638008, + "support": 15518.0 + }, + "1": { + "precision": 0.6566164154103853, + "recall": 0.2690152121697358, + "f1-score": 0.3816632860040568, + "support": 8743.0 + }, + "accuracy": 0.6858744487036808, + "macro avg": { + "precision": 0.6737794587328052, + "recall": 0.5948762102864402, + "f1-score": 0.5855604879339288, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6785723204660147, + "recall": 0.6858744487036808, + "f1-score": 0.642499754372859, + "support": 24261.0 + }, + "cross-entropy": 0.654021472689771, + "roc-auc": 0.6327357100453991, + "score": 0.6327357100453991 + }, + "test": { + "0": { + "precision": 0.6651265486363822, + "recall": 0.9612929913721783, + "f1-score": 0.7862442301650596, + "support": 16922.0 + }, + "1": { + "precision": 0.6721721721721722, + "recall": 0.14087905171509493, + "f1-score": 0.2329372994536467, + "support": 9533.0 + }, + "accuracy": 0.6656586656586656, + "macro avg": { + "precision": 0.6686493604042771, + "recall": 0.5510860215436366, + "f1-score": 0.5095907648093532, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6676654232977576, + "recall": 0.6656586656586656, + "f1-score": 0.5868613168983084, + "support": 26455.0 + }, + "cross-entropy": 0.7032628295376716, + "roc-auc": 0.5971884680332056, + "score": 0.5971884680332056 + } + }, + "time": "0:01:04.856616", + "diversity": { + "train": { + "mean_centered_corr": -0.009633250534534454, + "mean_pairwise_disagreement": 0.02490320677749485, + "member_std": 0.024462362751364708 + }, + "val": { + "mean_centered_corr": 0.023074615746736526, + "mean_pairwise_disagreement": 0.06982341565049974, + "member_std": 0.0468476265668869 + }, + "test": { + "mean_centered_corr": 0.04183991998434067, + "mean_pairwise_disagreement": 0.04013975344620506, + "member_std": 0.043004557490348816 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 15, + 10, + 9, + 20, + 9 + ], + "head_selection_time": "0:00:29.837541" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..eb29e4009 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2/report.json @@ -0,0 +1,220 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7869066099388643, + "recall": 0.9755402178690648, + "f1-score": 0.8711287439600714, + "support": 84179.0 + }, + "1": { + "precision": 0.5867951033513947, + "recall": 0.11620697877752166, + "f1-score": 0.19399568751036655, + "support": 25162.0 + }, + "accuracy": 0.7777869234779269, + "macro avg": { + "precision": 0.6868508566451295, + "recall": 0.5458735983232932, + "f1-score": 0.532562215735219, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7408561281547767, + "recall": 0.7777869234779269, + "f1-score": 0.7153041039221398, + "support": 109341.0 + }, + "cross-entropy": 0.4984563455304167, + "roc-auc": 0.6822619490208846, + "score": 0.6822619490208846 + }, + "val": { + "0": { + "precision": 0.6891333565632933, + "recall": 0.8921252738754994, + "f1-score": 0.7775999101300306, + "support": 15518.0 + }, + "1": { + "precision": 0.5987535953978907, + "recall": 0.2857142857142857, + "f1-score": 0.3868370112272551, + "support": 8743.0 + }, + "accuracy": 0.673591360619925, + "macro avg": { + "precision": 0.643943475980592, + "recall": 0.5889197797948926, + "f1-score": 0.5822184606786429, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6565629657356641, + "recall": 0.673591360619925, + "f1-score": 0.6367796626090312, + "support": 24261.0 + }, + "cross-entropy": 0.6333720055734767, + "roc-auc": 0.6356383175142475, + "score": 0.6356383175142475 + }, + "test": { + "0": { + "precision": 0.6669322375041932, + "recall": 0.9399007209549699, + "f1-score": 0.7802305616875154, + "support": 16922.0 + }, + "1": { + "precision": 0.6098964326812428, + "recall": 0.16678904856813176, + "f1-score": 0.26194398682042835, + "support": 9533.0 + }, + "accuracy": 0.6613116613116613, + "macro avg": { + "precision": 0.638414335092718, + "recall": 0.5533448847615509, + "f1-score": 0.5210872742539718, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.646379512976611, + "recall": 0.6613116613116613, + "f1-score": 0.5934671552158488, + "support": 26455.0 + }, + "cross-entropy": 0.662735308091, + "roc-auc": 0.5998927728985708, + "score": 0.5998927728985708 + } + }, + "time": "0:01:00.829886", + "diversity": { + "train": { + "mean_centered_corr": -0.0152942705899477, + "mean_pairwise_disagreement": 0.023205388676030093, + "member_std": 0.024158649146556854 + }, + "val": { + "mean_centered_corr": -0.00040229607839137316, + "mean_pairwise_disagreement": 0.06998355252223468, + "member_std": 0.05039098858833313 + }, + "test": { + "mean_centered_corr": 0.0061957864090800285, + "mean_pairwise_disagreement": 0.04117994037348876, + "member_std": 0.047571223229169846 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 20 + ], + "head_selection_time": "0:00:15.755507" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..72357a786 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/0/report.json @@ -0,0 +1,390 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.784937732846757, + "recall": 0.978628874184773, + "f1-score": 0.8711468302225982, + "support": 84179.0 + }, + "1": { + "precision": 0.5902050113895216, + "recall": 0.10297273666640172, + "f1-score": 0.17535192203573363, + "support": 25162.0 + }, + "accuracy": 0.7771192873670444, + "macro avg": { + "precision": 0.6875713721181393, + "recall": 0.5408008054255873, + "f1-score": 0.5232493761291659, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.740125039188322, + "recall": 0.7771192873670444, + "f1-score": 0.7110276482158681, + "support": 109341.0 + }, + "cross-entropy": 0.4989568858959861, + "roc-auc": 0.6810795630080747, + "score": 0.6810795630080747 + }, + "val": { + "0": { + "precision": 0.6882539682539682, + "recall": 0.9081067147828329, + "f1-score": 0.783041146889673, + "support": 15518.0 + }, + "1": { + "precision": 0.6233491811938722, + "recall": 0.26993022989820425, + "f1-score": 0.3767259956899992, + "support": 8743.0 + }, + "accuracy": 0.6781253864226536, + "macro avg": { + "precision": 0.6558015747239202, + "recall": 0.5890184723405185, + "f1-score": 0.579883571289836, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6648640604485843, + "recall": 0.6781253864226536, + "f1-score": 0.6366162935473232, + "support": 24261.0 + }, + "cross-entropy": 0.6298773246042463, + "roc-auc": 0.6410202011331968, + "score": 0.6410202011331968 + }, + "test": { + "0": { + "precision": 0.6683168316831684, + "recall": 0.9413780876964898, + "f1-score": 0.781687030766966, + "support": 16922.0 + }, + "1": { + "precision": 0.62122947689958, + "recall": 0.17067030315745305, + "f1-score": 0.26777485187623434, + "support": 9533.0 + }, + "accuracy": 0.6636552636552636, + "macro avg": { + "precision": 0.6447731542913742, + "recall": 0.5560241954269715, + "f1-score": 0.5247309413216001, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6513490088462018, + "recall": 0.6636552636552636, + "f1-score": 0.5964999280882533, + "support": 26455.0 + }, + "cross-entropy": 0.6607089123106477, + "roc-auc": 0.6024701571918213, + "score": 0.6024701571918213 + } + }, + "time": "0:01:11.748716", + "diversity": { + "train": { + "mean_centered_corr": -0.014211982488632202, + "mean_pairwise_disagreement": 0.02565772703979353, + "member_std": 0.02677466720342636 + }, + "val": { + "mean_centered_corr": -0.005956840701401234, + "mean_pairwise_disagreement": 0.08571943089333604, + "member_std": 0.0601416639983654 + }, + "test": { + "mean_centered_corr": 0.00013841522741131485, + "mean_pairwise_disagreement": 0.05374807189323318, + "member_std": 0.05852190777659416 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:04.721148" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..b203d8213 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/1/report.json @@ -0,0 +1,390 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.792594913777064, + "recall": 0.9615105905273287, + "f1-score": 0.8689196878120001, + "support": 84179.0 + }, + "1": { + "precision": 0.5513708114095819, + "recall": 0.15825451077020905, + "f1-score": 0.24592391304347827, + "support": 25162.0 + }, + "accuracy": 0.7766620023595906, + "macro avg": { + "precision": 0.671982862593323, + "recall": 0.5598825506487689, + "f1-score": 0.5574218004277391, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7370834325964402, + "recall": 0.7766620023595906, + "f1-score": 0.7255533413845342, + "support": 109341.0 + }, + "cross-entropy": 0.49701600203519025, + "roc-auc": 0.6913539415681078, + "score": 0.6913539415681078 + }, + "val": { + "0": { + "precision": 0.6963472629768289, + "recall": 0.8869699703570048, + "f1-score": 0.7801836526470922, + "support": 15518.0 + }, + "1": { + "precision": 0.6097886540600668, + "recall": 0.3135079492165161, + "f1-score": 0.41411089288412145, + "support": 8743.0 + }, + "accuracy": 0.6803099624912411, + "macro avg": { + "precision": 0.6530679585184478, + "recall": 0.6002389597867605, + "f1-score": 0.5971472727656069, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6651539107753842, + "recall": 0.6803099624912411, + "f1-score": 0.6482610551198817, + "support": 24261.0 + }, + "cross-entropy": 0.6721133601344941, + "roc-auc": 0.6322301668779651, + "score": 0.6322301668779651 + }, + "test": { + "0": { + "precision": 0.6682340213049268, + "recall": 0.9490013000827325, + "f1-score": 0.7842457391219417, + "support": 16922.0 + }, + "1": { + "precision": 0.6438299628559637, + "recall": 0.16364208538760097, + "f1-score": 0.2609568417530947, + "support": 9533.0 + }, + "accuracy": 0.665998865998866, + "macro avg": { + "precision": 0.6560319920804453, + "recall": 0.5563216927351667, + "f1-score": 0.5226012904375182, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6594400734994472, + "recall": 0.665998865998866, + "f1-score": 0.5956797569402287, + "support": 26455.0 + }, + "cross-entropy": 0.7354332507178938, + "roc-auc": 0.5946009422441442, + "score": 0.5946009422441442 + } + }, + "time": "0:01:16.578012", + "diversity": { + "train": { + "mean_centered_corr": -0.006815396714955568, + "mean_pairwise_disagreement": 0.024371262617009646, + "member_std": 0.024308200925588608 + }, + "val": { + "mean_centered_corr": 0.023836545646190643, + "mean_pairwise_disagreement": 0.06752075214302525, + "member_std": 0.04681727662682533 + }, + "test": { + "mean_centered_corr": 0.04357389360666275, + "mean_pairwise_disagreement": 0.038735893574603254, + "member_std": 0.043050214648246765 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 10 + ], + "head_selection_time": "0:00:04.824193" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..e54db3a3d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-best-head-evaluation/2/report.json @@ -0,0 +1,390 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7854278840646122, + "recall": 0.9773458938690173, + "f1-score": 0.8709395692516158, + "support": 84179.0 + }, + "1": { + "precision": 0.5848029610276507, + "recall": 0.1067482712026071, + "f1-score": 0.1805410855318434, + "support": 25162.0 + }, + "accuracy": 0.7770003932651064, + "macro avg": { + "precision": 0.6851154225461314, + "recall": 0.5420470825358121, + "f1-score": 0.5257403273917296, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7392592527784887, + "recall": 0.7770003932651064, + "f1-score": 0.7120622346071831, + "support": 109341.0 + }, + "cross-entropy": 0.4995940703011633, + "roc-auc": 0.6793517558838738, + "score": 0.6793517558838738 + }, + "val": { + "0": { + "precision": 0.6817984256531607, + "recall": 0.9097821884263436, + "f1-score": 0.7794616977225672, + "support": 15518.0 + }, + "1": { + "precision": 0.6060776589758019, + "recall": 0.24636852339014068, + "f1-score": 0.350329348621615, + "support": 8743.0 + }, + "accuracy": 0.670706071472734, + "macro avg": { + "precision": 0.6439380423144814, + "recall": 0.5780753559082421, + "f1-score": 0.5648955231720911, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6545107349948965, + "recall": 0.670706071472734, + "f1-score": 0.6248141511173314, + "support": 24261.0 + }, + "cross-entropy": 0.6449115869812975, + "roc-auc": 0.6333770862914994, + "score": 0.6333770862914994 + }, + "test": { + "0": { + "precision": 0.6628663819907964, + "recall": 0.9448646732064768, + "f1-score": 0.7791340788928685, + "support": 16922.0 + }, + "1": { + "precision": 0.6002570694087404, + "recall": 0.1469631805307878, + "f1-score": 0.2361169630066571, + "support": 9533.0 + }, + "accuracy": 0.6573426573426573, + "macro avg": { + "precision": 0.6315617256997683, + "recall": 0.5459139268686323, + "f1-score": 0.5076255209497628, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6403052564249395, + "recall": 0.6573426573426573, + "f1-score": 0.583459077352772, + "support": 26455.0 + }, + "cross-entropy": 0.6839435417360092, + "roc-auc": 0.5972389647476771, + "score": 0.5972389647476771 + } + }, + "time": "0:01:12.365895", + "diversity": { + "train": { + "mean_centered_corr": -0.01532948762178421, + "mean_pairwise_disagreement": 0.02282787040011848, + "member_std": 0.0240154005587101 + }, + "val": { + "mean_centered_corr": 0.0001475554599892348, + "mean_pairwise_disagreement": 0.07275033872230886, + "member_std": 0.05133147910237312 + }, + "test": { + "mean_centered_corr": 0.007742947433143854, + "mean_pairwise_disagreement": 0.042625182141311176, + "member_std": 0.048414889723062515 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 20 + ], + "head_selection_time": "0:00:04.822432" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..0e6ac6970 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..0ed76a5a7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,386 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7863951555903647, + "recall": 0.97808241960584, + "f1-score": 0.8718266385001879, + "support": 84179.0 + }, + "1": { + "precision": 0.602627611458109, + "recall": 0.11119942770844925, + "f1-score": 0.18775373259520214, + "support": 25162.0 + }, + "accuracy": 0.7785917450910455, + "macro avg": { + "precision": 0.6945113835242368, + "recall": 0.5446409236571446, + "f1-score": 0.5297901855476951, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441058135735931, + "recall": 0.7785917450910455, + "f1-score": 0.7144049718025973, + "support": 109341.0 + }, + "cross-entropy": 0.49233822908120295, + "roc-auc": 0.6988844522375441, + "score": 0.6988844522375441 + }, + "val": { + "0": { + "precision": 0.6753571753571753, + "recall": 0.9565021265627014, + "f1-score": 0.7917111158523575, + "support": 15518.0 + }, + "1": { + "precision": 0.7043363994743759, + "recall": 0.1839185634221663, + "f1-score": 0.2916742245601306, + "support": 8743.0 + }, + "accuracy": 0.6780841680062653, + "macro avg": { + "precision": 0.6898467874157757, + "recall": 0.5702103449924338, + "f1-score": 0.5416926702062441, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6858004941180131, + "recall": 0.6780841680062653, + "f1-score": 0.6115115139988503, + "support": 24261.0 + }, + "cross-entropy": 0.6866508172905287, + "roc-auc": 0.6228186386127663, + "score": 0.6228186386127663 + }, + "test": { + "0": { + "precision": 0.6564383127180463, + "recall": 0.97848954024347, + "f1-score": 0.7857447919138234, + "support": 16922.0 + }, + "1": { + "precision": 0.7043054427294883, + "recall": 0.09094723591733976, + "f1-score": 0.16109253065774806, + "support": 9533.0 + }, + "accuracy": 0.6586656586656586, + "macro avg": { + "precision": 0.6803718777237673, + "recall": 0.5347183880804048, + "f1-score": 0.47341866128578575, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6736871258119445, + "recall": 0.6586656586656586, + "f1-score": 0.5606527485740326, + "support": 26455.0 + }, + "cross-entropy": 0.7554621625639649, + "roc-auc": 0.5911736900637132, + "score": 0.5911736900637132 + } + }, + "time": "0:01:11.748716", + "diversity": { + "train": { + "mean_centered_corr": -0.014211982488632202, + "mean_pairwise_disagreement": 0.02565772703979353, + "member_std": 0.02677466720342636 + }, + "val": { + "mean_centered_corr": -0.005956840701401234, + "mean_pairwise_disagreement": 0.08571943089333604, + "member_std": 0.0601416639983654 + }, + "test": { + "mean_centered_corr": 0.00013841522741131485, + "mean_pairwise_disagreement": 0.05374807189323318, + "member_std": 0.05852190777659416 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml new file mode 100644 index 000000000..02ae43937 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1/report.json new file mode 100644 index 000000000..37db253a4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1/report.json @@ -0,0 +1,386 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7883172407167219, + "recall": 0.9747324154480334, + "f1-score": 0.8716695703905154, + "support": 84179.0 + }, + "1": { + "precision": 0.5953196347031964, + "recall": 0.12435418488196487, + "f1-score": 0.20573344730094023, + "support": 25162.0 + }, + "accuracy": 0.7790398843983501, + "macro avg": { + "precision": 0.6918184377099592, + "recall": 0.5495433001649991, + "f1-score": 0.5387015088457279, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439038389505744, + "recall": 0.7790398843983501, + "f1-score": 0.7184216146449133, + "support": 109341.0 + }, + "cross-entropy": 0.4903796076721471, + "roc-auc": 0.7031730821629574, + "score": 0.7031730821629574 + }, + "val": { + "0": { + "precision": 0.6731283119706508, + "recall": 0.9577265111483438, + "f1-score": 0.7905949942814586, + "support": 15518.0 + }, + "1": { + "precision": 0.6993583868011, + "recall": 0.1745396317053643, + "f1-score": 0.2793592677345538, + "support": 8743.0 + }, + "accuracy": 0.6754874077737933, + "macro avg": { + "precision": 0.6862433493858754, + "recall": 0.5661330714268541, + "f1-score": 0.5349771310080063, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6825809126154146, + "recall": 0.6754874077737933, + "f1-score": 0.6063596388880459, + "support": 24261.0 + }, + "cross-entropy": 0.7146728501800901, + "roc-auc": 0.6202479078617598, + "score": 0.6202479078617598 + }, + "test": { + "0": { + "precision": 0.6551465063861758, + "recall": 0.979080486940078, + "f1-score": 0.7850086470351331, + "support": 16922.0 + }, + "1": { + "precision": 0.6963979416809606, + "recall": 0.0851778034197, + "f1-score": 0.15178988690531825, + "support": 9533.0 + }, + "accuracy": 0.656964656964657, + "macro avg": { + "precision": 0.6757722240335682, + "recall": 0.532129145179889, + "f1-score": 0.46839926697022566, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.670011368705782, + "recall": 0.656964656964657, + "f1-score": 0.5568296850121686, + "support": 26455.0 + }, + "cross-entropy": 0.7991918340623438, + "roc-auc": 0.5895844228260869, + "score": 0.5895844228260869 + } + }, + "time": "0:01:16.578012", + "diversity": { + "train": { + "mean_centered_corr": -0.006815396714955568, + "mean_pairwise_disagreement": 0.024371262617009646, + "member_std": 0.024308200925588608 + }, + "val": { + "mean_centered_corr": 0.023836545646190643, + "mean_pairwise_disagreement": 0.06752075214302525, + "member_std": 0.04681727662682533 + }, + "test": { + "mean_centered_corr": 0.04357389360666275, + "mean_pairwise_disagreement": 0.038735893574603254, + "member_std": 0.043050214648246765 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml new file mode 100644 index 000000000..abc9c53f7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2/report.json new file mode 100644 index 000000000..044a1ff43 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2/report.json @@ -0,0 +1,386 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7860148668377912, + "recall": 0.9785338386058281, + "f1-score": 0.871772076877487, + "support": 84179.0 + }, + "1": { + "precision": 0.6023327464788732, + "recall": 0.10877513711151737, + "f1-score": 0.18427253753450482, + "support": 25162.0 + }, + "accuracy": 0.7783813939876167, + "macro avg": { + "precision": 0.6941738066583323, + "recall": 0.5436544878586728, + "f1-score": 0.528022307205996, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7437451828905884, + "recall": 0.7783813939876167, + "f1-score": 0.7135618592194436, + "support": 109341.0 + }, + "cross-entropy": 0.491969658143707, + "roc-auc": 0.6998568314138789, + "score": 0.6998568314138789 + }, + "val": { + "0": { + "precision": 0.676559218336225, + "recall": 0.9548910942131718, + "f1-score": 0.791982896846606, + "support": 15518.0 + }, + "1": { + "precision": 0.7032640949554896, + "recall": 0.18975180144115292, + "f1-score": 0.2988650693568726, + "support": 8743.0 + }, + "accuracy": 0.6791558468323647, + "macro avg": { + "precision": 0.6899116566458573, + "recall": 0.5723214478271623, + "f1-score": 0.5454239831017393, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6861829245429861, + "recall": 0.6791558468323647, + "f1-score": 0.614276736105386, + "support": 24261.0 + }, + "cross-entropy": 0.7019446862265161, + "roc-auc": 0.6211436845976699, + "score": 0.6211436845976699 + }, + "test": { + "0": { + "precision": 0.65782767956681, + "recall": 0.9763621321356813, + "f1-score": 0.7860507160188401, + "support": 16922.0 + }, + "1": { + "precision": 0.7012696041822255, + "recall": 0.09849994755061366, + "f1-score": 0.1727373068432671, + "support": 9533.0 + }, + "accuracy": 0.66002646002646, + "macro avg": { + "precision": 0.6795486418745178, + "recall": 0.5374310398431476, + "f1-score": 0.47939401143105365, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6734818798071712, + "recall": 0.66002646002646, + "f1-score": 0.5650446026311728, + "support": 26455.0 + }, + "cross-entropy": 0.7793654776091676, + "roc-auc": 0.5891830464738508, + "score": 0.5891830464738508 + } + }, + "time": "0:01:12.365895", + "diversity": { + "train": { + "mean_centered_corr": -0.01532948762178421, + "mean_pairwise_disagreement": 0.02282787040011848, + "member_std": 0.0240154005587101 + }, + "val": { + "mean_centered_corr": 0.0001475554599892348, + "mean_pairwise_disagreement": 0.07275033872230886, + "member_std": 0.05133147910237312 + }, + "test": { + "mean_centered_corr": 0.007742947433143854, + "mean_pairwise_disagreement": 0.042625182141311176, + "member_std": 0.048414889723062515 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..edf46fa0f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,391 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7827118483860296, + "recall": 0.9837013982109553, + "f1-score": 0.8717719266847043, + "support": 84179.0 + }, + "1": { + "precision": 0.6130851663846588, + "recall": 0.08640012717590016, + "f1-score": 0.15145604012818725, + "support": 25162.0 + }, + "accuracy": 0.7772107443685351, + "macro avg": { + "precision": 0.6978985073853442, + "recall": 0.5350507626934277, + "f1-score": 0.5116139834064457, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436766596414737, + "recall": 0.7772107443685351, + "f1-score": 0.7060098764241883, + "support": 109341.0 + }, + "cross-entropy": 0.4979833881789, + "roc-auc": 0.6847415093580901, + "score": 0.6847415093580901 + }, + "val": { + "0": { + "precision": 0.6773795373502789, + "recall": 0.9548266529191907, + "f1-score": 0.7925224646983312, + "support": 15518.0 + }, + "1": { + "precision": 0.7063259321323837, + "recall": 0.19283998627473406, + "f1-score": 0.30296495956873315, + "support": 8743.0 + }, + "accuracy": 0.6802275256584642, + "macro avg": { + "precision": 0.6918527347413312, + "recall": 0.5738333195969624, + "f1-score": 0.5477437121335322, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6878110253177965, + "recall": 0.6802275256584642, + "f1-score": 0.6160993466344395, + "support": 24261.0 + }, + "cross-entropy": 0.6392793150076335, + "roc-auc": 0.6419149054445072, + "score": 0.6419149054445072 + }, + "test": { + "0": { + "precision": 0.658847933620552, + "recall": 0.9760075641177166, + "f1-score": 0.7866634913074542, + "support": 16922.0 + }, + "1": { + "precision": 0.7072819033886085, + "recall": 0.10290569600335676, + "f1-score": 0.17967032967032967, + "support": 9533.0 + }, + "accuracy": 0.6613872613872613, + "macro avg": { + "precision": 0.6830649185045803, + "recall": 0.5394566300605367, + "f1-score": 0.48316691048889193, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6763010061512224, + "recall": 0.6613872613872613, + "f1-score": 0.5679348649651103, + "support": 26455.0 + }, + "cross-entropy": 0.6803501878150763, + "roc-auc": 0.6041224275423289, + "score": 0.6041224275423289 + } + }, + "time": "0:01:11.748716", + "diversity": { + "train": { + "mean_centered_corr": -0.014211982488632202, + "mean_pairwise_disagreement": 0.02565772703979353, + "member_std": 0.02677466720342636 + }, + "val": { + "mean_centered_corr": -0.005956840701401234, + "mean_pairwise_disagreement": 0.08571943089333604, + "member_std": 0.0601416639983654 + }, + "test": { + "mean_centered_corr": 0.00013841522741131485, + "mean_pairwise_disagreement": 0.05374807189323318, + "member_std": 0.05852190777659416 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 25 + ], + "head_selection_time": "0:00:15.652537" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..b23c33390 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,392 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7934533487433643, + "recall": 0.9623540312904644, + "f1-score": 0.8697799513627554, + "support": 84179.0 + }, + "1": { + "precision": 0.5624741129366285, + "recall": 0.16191081790000794, + "f1-score": 0.2514426785989816, + "support": 25162.0 + }, + "accuracy": 0.7781527514838898, + "macro avg": { + "precision": 0.6779637308399964, + "recall": 0.5621324245952362, + "f1-score": 0.5606113149808685, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7402994583329137, + "recall": 0.7781527514838898, + "f1-score": 0.7274856385497934, + "support": 109341.0 + }, + "cross-entropy": 0.49515974918562666, + "roc-auc": 0.6937247371184572, + "score": 0.6937247371184572 + }, + "val": { + "0": { + "precision": 0.6973154362416107, + "recall": 0.8704085578038407, + "f1-score": 0.7743063517541848, + "support": 15518.0 + }, + "1": { + "precision": 0.5888366387241872, + "recall": 0.3294063822486561, + "f1-score": 0.42247322869297343, + "support": 8743.0 + }, + "accuracy": 0.6754461893574049, + "macro avg": { + "precision": 0.643076037482899, + "recall": 0.5999074700262484, + "f1-score": 0.5983897902235791, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.65822264836416, + "recall": 0.6754461893574049, + "f1-score": 0.6475153293344919, + "support": 24261.0 + }, + "cross-entropy": 0.6446266565354483, + "roc-auc": 0.6333421643138163, + "score": 0.6333421643138163 + }, + "test": { + "0": { + "precision": 0.669557164505809, + "recall": 0.9399598156246307, + "f1-score": 0.7820443482963764, + "support": 16922.0 + }, + "1": { + "precision": 0.6235642830678029, + "recall": 0.1765446344277772, + "f1-score": 0.2751798561151079, + "support": 9533.0 + }, + "accuracy": 0.6648648648648648, + "macro avg": { + "precision": 0.646560723786806, + "recall": 0.558252225026204, + "f1-score": 0.5286121022057422, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6529837326876835, + "recall": 0.664864864864865, + "f1-score": 0.5993968637390514, + "support": 26455.0 + }, + "cross-entropy": 0.682979018710851, + "roc-auc": 0.5975759215250558, + "score": 0.5975759215250558 + } + }, + "time": "0:01:16.578012", + "diversity": { + "train": { + "mean_centered_corr": -0.006815396714955568, + "mean_pairwise_disagreement": 0.024371262617009646, + "member_std": 0.024308200925588608 + }, + "val": { + "mean_centered_corr": 0.023836545646190643, + "mean_pairwise_disagreement": 0.06752075214302525, + "member_std": 0.04681727662682533 + }, + "test": { + "mean_centered_corr": 0.04357389360666275, + "mean_pairwise_disagreement": 0.038735893574603254, + "member_std": 0.043050214648246765 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 10, + 9, + 11 + ], + "head_selection_time": "0:00:20.590968" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..e79af5c4a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,392 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7875746118265743, + "recall": 0.9749462455006593, + "f1-score": 0.8713007935875998, + "support": 84179.0 + }, + "1": { + "precision": 0.5892891918208374, + "recall": 0.12026071059534219, + "f1-score": 0.1997557513945275, + "support": 25162.0 + }, + "accuracy": 0.7782624998856787, + "macro avg": { + "precision": 0.6884319018237058, + "recall": 0.5476034780480008, + "f1-score": 0.5355282724910637, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7419443565866886, + "recall": 0.7782624998856787, + "f1-score": 0.7167620903412231, + "support": 109341.0 + }, + "cross-entropy": 0.4966763535938117, + "roc-auc": 0.6873519813280431, + "score": 0.6873519813280431 + }, + "val": { + "0": { + "precision": 0.6921494686321563, + "recall": 0.9107488078360614, + "f1-score": 0.7865431171216295, + "support": 15518.0 + }, + "1": { + "precision": 0.6395106715252473, + "recall": 0.28102481985588473, + "f1-score": 0.3904648390941597, + "support": 8743.0 + }, + "accuracy": 0.6838135278842586, + "macro avg": { + "precision": 0.6658300700787019, + "recall": 0.5958868138459731, + "f1-score": 0.5885039781078946, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6731798876954388, + "recall": 0.6838135278842586, + "f1-score": 0.6438073525284895, + "support": 24261.0 + }, + "cross-entropy": 0.6379588554594992, + "roc-auc": 0.6340638176219543, + "score": 0.6340638176219543 + }, + "test": { + "0": { + "precision": 0.668311699083482, + "recall": 0.9523106015837371, + "f1-score": 0.7854270744486413, + "support": 16922.0 + }, + "1": { + "precision": 0.6554227156276686, + "recall": 0.16101961607049198, + "f1-score": 0.25852631578947366, + "support": 9533.0 + }, + "accuracy": 0.6671706671706672, + "macro avg": { + "precision": 0.6618672073555754, + "recall": 0.5566651088271146, + "f1-score": 0.5219766951190575, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6636671827620203, + "recall": 0.6671706671706672, + "f1-score": 0.5955595661402745, + "support": 26455.0 + }, + "cross-entropy": 0.6753783561951368, + "roc-auc": 0.598916340879379, + "score": 0.598916340879379 + } + }, + "time": "0:01:12.365895", + "diversity": { + "train": { + "mean_centered_corr": -0.01532948762178421, + "mean_pairwise_disagreement": 0.02282787040011848, + "member_std": 0.0240154005587101 + }, + "val": { + "mean_centered_corr": 0.0001475554599892348, + "mean_pairwise_disagreement": 0.07275033872230886, + "member_std": 0.05133147910237312 + }, + "test": { + "mean_centered_corr": 0.007742947433143854, + "mean_pairwise_disagreement": 0.042625182141311176, + "member_std": 0.048414889723062515 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 20, + 25, + 31 + ], + "head_selection_time": "0:00:20.602844" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8fc29a25e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/0/report.json @@ -0,0 +1,426 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7889205040357703, + "recall": 0.9683650316587272, + "f1-score": 0.8694808138449641, + "support": 84179.0 + }, + "1": { + "precision": 0.5572734829592685, + "recall": 0.13321675542484698, + "f1-score": 0.21503031080604293, + "support": 25162.0 + }, + "accuracy": 0.7761772802516896, + "macro avg": { + "precision": 0.6730969934975194, + "recall": 0.5507908935417871, + "f1-score": 0.5422555623255035, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7356129401363461, + "recall": 0.7761772802516896, + "f1-score": 0.7188759761585946, + "support": 109341.0 + }, + "cross-entropy": 0.4988956148214662, + "roc-auc": 0.68250001764071, + "score": 0.68250001764071 + }, + "val": { + "0": { + "precision": 0.6950127877237852, + "recall": 0.910619925248099, + "f1-score": 0.7883403068340307, + "support": 15518.0 + }, + "1": { + "precision": 0.6469839653855943, + "recall": 0.2907468832208624, + "f1-score": 0.401199494949495, + "support": 8743.0 + }, + "accuracy": 0.6872346564444994, + "macro avg": { + "precision": 0.6709983765546897, + "recall": 0.6006834042344806, + "f1-score": 0.5947699008917628, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6777045154471766, + "recall": 0.6872346564444994, + "f1-score": 0.6488253602816835, + "support": 24261.0 + }, + "cross-entropy": 0.6245548911838499, + "roc-auc": 0.6477890245840551, + "score": 0.6477890245840551 + }, + "test": { + "0": { + "precision": 0.6679904029122198, + "recall": 0.9542607256825434, + "f1-score": 0.7858672376873662, + "support": 16922.0 + }, + "1": { + "precision": 0.6606751424813678, + "recall": 0.1580824504353299, + "f1-score": 0.25512104283054005, + "support": 9533.0 + }, + "accuracy": 0.6673596673596673, + "macro avg": { + "precision": 0.6643327726967938, + "recall": 0.5561715880589366, + "f1-score": 0.5204941402589531, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6653543651996018, + "recall": 0.6673596673596673, + "f1-score": 0.5946140350576129, + "support": 26455.0 + }, + "cross-entropy": 0.6584188876218103, + "roc-auc": 0.6045088303107439, + "score": 0.6045088303107439 + } + }, + "time": "0:01:18.654365", + "diversity": { + "train": { + "mean_centered_corr": -0.01063462533056736, + "mean_pairwise_disagreement": 0.026665487313881315, + "member_std": 0.029800426214933395 + }, + "val": { + "mean_centered_corr": -0.013281173072755337, + "mean_pairwise_disagreement": 0.09412748922670262, + "member_std": 0.06749773025512695 + }, + "test": { + "mean_centered_corr": -0.011601177044212818, + "mean_pairwise_disagreement": 0.059119563958273634, + "member_std": 0.06550896167755127 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:04.756683" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/1/report.json new file mode 100644 index 000000000..990349219 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/1/report.json @@ -0,0 +1,426 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9005681872367859, + "min_features_per_member": 89.0, + "mean_features_per_member": 99.0625, + "max_features_per_member": 106.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.81818199157715, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8186085273842617, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7888152627873106, + "recall": 0.9715249646586441, + "f1-score": 0.8706881866961929, + "support": 84179.0 + }, + "1": { + "precision": 0.5768008474576272, + "recall": 0.12983864557666322, + "f1-score": 0.21196392655550508, + "support": 25162.0 + }, + "accuracy": 0.7778326519786722, + "macro avg": { + "precision": 0.6828080551224689, + "recall": 0.5506818051176536, + "f1-score": 0.541326056625849, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.740025634756421, + "recall": 0.7778326519786722, + "f1-score": 0.7190998544725988, + "support": 109341.0 + }, + "cross-entropy": 0.497526106332481, + "roc-auc": 0.6852792162881653, + "score": 0.6852792162881653 + }, + "val": { + "0": { + "precision": 0.6946272468323348, + "recall": 0.9114576620698543, + "f1-score": 0.7884057971014493, + "support": 15518.0 + }, + "1": { + "precision": 0.6476019492177482, + "recall": 0.28880247054786684, + "f1-score": 0.39946211042556556, + "support": 8743.0 + }, + "accuracy": 0.6870697827789456, + "macro avg": { + "precision": 0.6711145980250415, + "recall": 0.6001300663088606, + "f1-score": 0.5939339537635074, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6776806173840709, + "recall": 0.6870697827789456, + "f1-score": 0.6482411438469564, + "support": 24261.0 + }, + "cross-entropy": 0.6172041286027814, + "roc-auc": 0.6512227217747169, + "score": 0.6512227217747169 + }, + "test": { + "0": { + "precision": 0.6737023929895517, + "recall": 0.9449828625457983, + "f1-score": 0.7866102612032072, + "support": 16922.0 + }, + "1": { + "precision": 0.6575947039352703, + "recall": 0.18755900555963495, + "f1-score": 0.29187071498530853, + "support": 9533.0 + }, + "accuracy": 0.6720468720468721, + "macro avg": { + "precision": 0.6656485484624111, + "recall": 0.5662709340527167, + "f1-score": 0.5392404880942578, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6678980233144632, + "recall": 0.6720468720468721, + "f1-score": 0.6083319737681201, + "support": 26455.0 + }, + "cross-entropy": 0.6427700166976323, + "roc-auc": 0.6075381341628896, + "score": 0.6075381341628896 + } + }, + "time": "0:01:23.222825", + "diversity": { + "train": { + "mean_centered_corr": -0.008371543139219284, + "mean_pairwise_disagreement": 0.02743249071932702, + "member_std": 0.027091743424534798 + }, + "val": { + "mean_centered_corr": 0.05517549440264702, + "mean_pairwise_disagreement": 0.07159946402762432, + "member_std": 0.054152194410562515 + }, + "test": { + "mean_centered_corr": 0.09162398427724838, + "mean_pairwise_disagreement": 0.04175463812560587, + "member_std": 0.052940964698791504 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:04.823752" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/2/report.json new file mode 100644 index 000000000..e1b081026 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-best-head-evaluation/2/report.json @@ -0,0 +1,426 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8940340876579285, + "min_features_per_member": 89.0, + "mean_features_per_member": 98.34375, + "max_features_per_member": 104.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.60909080505371, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8074834695307933, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.788166502634169, + "recall": 0.9703607788165695, + "f1-score": 0.8698254151647614, + "support": 84179.0 + }, + "1": { + "precision": 0.56251095914431, + "recall": 0.12749383991733568, + "f1-score": 0.20787299530212214, + "support": 25162.0 + }, + "accuracy": 0.7763967770552674, + "macro avg": { + "precision": 0.6753387308892396, + "recall": 0.5489273093669526, + "f1-score": 0.5388492052334418, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7362377221648864, + "recall": 0.7763967770552674, + "f1-score": 0.7174942055674125, + "support": 109341.0 + }, + "cross-entropy": 0.5009419379304306, + "roc-auc": 0.6745540773335442, + "score": 0.6745540773335442 + }, + "val": { + "0": { + "precision": 0.6869277961302501, + "recall": 0.9380074751901019, + "f1-score": 0.7930696305982348, + "support": 15518.0 + }, + "1": { + "precision": 0.6867469879518072, + "recall": 0.24122154866750542, + "f1-score": 0.3570340274250889, + "support": 8743.0 + }, + "accuracy": 0.6869049091133919, + "macro avg": { + "precision": 0.6868373920410287, + "recall": 0.5896145119288037, + "f1-score": 0.5750518290116619, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6868626378142645, + "recall": 0.6869049091133919, + "f1-score": 0.6359343402745542, + "support": 24261.0 + }, + "cross-entropy": 0.6325180503678897, + "roc-auc": 0.6430199007953441, + "score": 0.6430199007953441 + }, + "test": { + "0": { + "precision": 0.6649977612244067, + "recall": 0.965429618248434, + "f1-score": 0.7875340451685989, + "support": 16922.0 + }, + "1": { + "precision": 0.6901483050847458, + "recall": 0.13668310080772056, + "f1-score": 0.22817616671044566, + "support": 9533.0 + }, + "accuracy": 0.6667926667926668, + "macro avg": { + "precision": 0.6775730331545762, + "recall": 0.5510563595280773, + "f1-score": 0.5078551059395223, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.674060703376008, + "recall": 0.6667926667926668, + "f1-score": 0.5859706864333286, + "support": 26455.0 + }, + "cross-entropy": 0.6784012446842685, + "roc-auc": 0.5970664756329549, + "score": 0.5970664756329549 + } + }, + "time": "0:01:18.718791", + "diversity": { + "train": { + "mean_centered_corr": -0.007062356453388929, + "mean_pairwise_disagreement": 0.025052193183149135, + "member_std": 0.028095407411456108 + }, + "val": { + "mean_centered_corr": -0.003141294466331601, + "mean_pairwise_disagreement": 0.09142868017301098, + "member_std": 0.06040506064891815 + }, + "test": { + "mean_centered_corr": 0.00333536509424448, + "mean_pairwise_disagreement": 0.05701769895318282, + "member_std": 0.059019479900598526 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 23 + ], + "head_selection_time": "0:00:04.822535" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml new file mode 100644 index 000000000..cd8d74155 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml @@ -0,0 +1,57 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/0/report.json new file mode 100644 index 000000000..5e1e2aae3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/0/report.json @@ -0,0 +1,422 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7842480378851866, + "recall": 0.9816818921583768, + "f1-score": 0.8719282511210762, + "support": 84179.0 + }, + "1": { + "precision": 0.6115869017632242, + "recall": 0.09649471425164931, + "f1-score": 0.1666895510091995, + "support": 25162.0 + }, + "accuracy": 0.7779789831810574, + "macro avg": { + "precision": 0.6979174698242054, + "recall": 0.539088303205013, + "f1-score": 0.5193089010651378, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7445145480954387, + "recall": 0.7779789831810574, + "f1-score": 0.7096358249294825, + "support": 109341.0 + }, + "cross-entropy": 0.4927771605769271, + "roc-auc": 0.6981357694476362, + "score": 0.6981357694476362 + }, + "val": { + "0": { + "precision": 0.6767607563716087, + "recall": 0.9548266529191907, + "f1-score": 0.7920987918314979, + "support": 15518.0 + }, + "1": { + "precision": 0.7038445289395859, + "recall": 0.19055244195356286, + "f1-score": 0.29990999099909993, + "support": 8743.0 + }, + "accuracy": 0.6794031573306953, + "macro avg": { + "precision": 0.6903026426555974, + "recall": 0.5726895474363768, + "f1-score": 0.5460043914152989, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6865210063020248, + "recall": 0.6794031573306953, + "f1-score": 0.6147274268557072, + "support": 24261.0 + }, + "cross-entropy": 0.673915591317614, + "roc-auc": 0.6250550271749447, + "score": 0.6250550271749447 + }, + "test": { + "0": { + "precision": 0.657534791252485, + "recall": 0.9772485521805933, + "f1-score": 0.7861285415478227, + "support": 16922.0 + }, + "1": { + "precision": 0.7049808429118773, + "recall": 0.09650687086961082, + "f1-score": 0.16977302085255583, + "support": 9533.0 + }, + "accuracy": 0.6598752598752599, + "macro avg": { + "precision": 0.6812578170821812, + "recall": 0.536877711525102, + "f1-score": 0.4779507812001893, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6746318697052912, + "recall": 0.6598752598752599, + "f1-score": 0.5640262100873057, + "support": 26455.0 + }, + "cross-entropy": 0.735759481729751, + "roc-auc": 0.592670158895295, + "score": 0.592670158895295 + } + }, + "time": "0:01:18.654365", + "diversity": { + "train": { + "mean_centered_corr": -0.01063462533056736, + "mean_pairwise_disagreement": 0.026665487313881315, + "member_std": 0.029800426214933395 + }, + "val": { + "mean_centered_corr": -0.013281173072755337, + "mean_pairwise_disagreement": 0.09412748922670262, + "member_std": 0.06749773025512695 + }, + "test": { + "mean_centered_corr": -0.011601177044212818, + "mean_pairwise_disagreement": 0.059119563958273634, + "member_std": 0.06550896167755127 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml new file mode 100644 index 000000000..9e8dabb63 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml @@ -0,0 +1,57 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 1 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/1/report.json new file mode 100644 index 000000000..6da5b519a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/1/report.json @@ -0,0 +1,422 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9005681872367859, + "min_features_per_member": 89.0, + "mean_features_per_member": 99.0625, + "max_features_per_member": 106.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.81818199157715, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8186085273842617, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7861034512370829, + "recall": 0.9787001508689815, + "f1-score": 0.8718925611963044, + "support": 84179.0 + }, + "1": { + "precision": 0.6048920229175848, + "recall": 0.10909307686193466, + "f1-score": 0.18484848484848485, + "support": 25162.0 + }, + "accuracy": 0.7785825993908964, + "macro avg": { + "precision": 0.6954977370773339, + "recall": 0.5438966138654581, + "f1-score": 0.5283705230223946, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7444023330894968, + "recall": 0.7785825993908964, + "f1-score": 0.7137871565533632, + "support": 109341.0 + }, + "cross-entropy": 0.49057469501963574, + "roc-auc": 0.7028790214614515, + "score": 0.7028790214614515 + }, + "val": { + "0": { + "precision": 0.6765809462798572, + "recall": 0.9528289728057739, + "f1-score": 0.7912875949909023, + "support": 15518.0 + }, + "1": { + "precision": 0.6958869962609057, + "recall": 0.1915818368980899, + "f1-score": 0.3004484304932735, + "support": 8743.0 + }, + "accuracy": 0.6784963521701496, + "macro avg": { + "precision": 0.6862339712703814, + "recall": 0.5722054048519319, + "f1-score": 0.5458680127420878, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6835383179868894, + "recall": 0.6784963521701496, + "f1-score": 0.6144026019896752, + "support": 24261.0 + }, + "cross-entropy": 0.6967356655889665, + "roc-auc": 0.6213201555665758, + "score": 0.6213201555665758 + }, + "test": { + "0": { + "precision": 0.6572702659659181, + "recall": 0.9755348067604303, + "f1-score": 0.7853846519815405, + "support": 16922.0 + }, + "1": { + "precision": 0.6908140403286034, + "recall": 0.09703136473303263, + "f1-score": 0.17016188373804267, + "support": 9533.0 + }, + "accuracy": 0.658968058968059, + "macro avg": { + "precision": 0.6740421531472607, + "recall": 0.5362830857467314, + "f1-score": 0.4777732678597916, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6693576899311223, + "recall": 0.658968058968059, + "f1-score": 0.5636905053300468, + "support": 26455.0 + }, + "cross-entropy": 0.7710291094729679, + "roc-auc": 0.5900624771932574, + "score": 0.5900624771932574 + } + }, + "time": "0:01:23.222825", + "diversity": { + "train": { + "mean_centered_corr": -0.008371543139219284, + "mean_pairwise_disagreement": 0.02743249071932702, + "member_std": 0.027091743424534798 + }, + "val": { + "mean_centered_corr": 0.05517549440264702, + "mean_pairwise_disagreement": 0.07159946402762432, + "member_std": 0.054152194410562515 + }, + "test": { + "mean_centered_corr": 0.09162398427724838, + "mean_pairwise_disagreement": 0.04175463812560587, + "member_std": 0.052940964698791504 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml new file mode 100644 index 000000000..9551165d5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml @@ -0,0 +1,57 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 2 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/2/report.json new file mode 100644 index 000000000..254ffb5e5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-evaluation/2/report.json @@ -0,0 +1,422 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8940340876579285, + "min_features_per_member": 89.0, + "mean_features_per_member": 98.34375, + "max_features_per_member": 104.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.60909080505371, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8074834695307933, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7848450275717818, + "recall": 0.9806483802373513, + "f1-score": 0.8718888460543202, + "support": 84179.0 + }, + "1": { + "precision": 0.608507570295602, + "recall": 0.10062793100707416, + "f1-score": 0.17269720015005285, + "support": 25162.0 + }, + "accuracy": 0.7781344600835917, + "macro avg": { + "precision": 0.6966762989336919, + "recall": 0.5406381556222127, + "f1-score": 0.5222930231021865, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7442655276588193, + "recall": 0.7781344600835917, + "f1-score": 0.7109879928131465, + "support": 109341.0 + }, + "cross-entropy": 0.4923260014405221, + "roc-auc": 0.699065450692943, + "score": 0.699065450692943 + }, + "val": { + "0": { + "precision": 0.678972564905174, + "recall": 0.9505090862224513, + "f1-score": 0.7921164276891681, + "support": 15518.0 + }, + "1": { + "precision": 0.6972802522664564, + "recall": 0.20233329520759466, + "f1-score": 0.31365248226950354, + "support": 8743.0 + }, + "accuracy": 0.6808870203206793, + "macro avg": { + "precision": 0.6881264085858152, + "recall": 0.576421190715023, + "f1-score": 0.5528844549793358, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6855701540647177, + "recall": 0.6808870203206793, + "f1-score": 0.6196911247418813, + "support": 24261.0 + }, + "cross-entropy": 0.6857647009535869, + "roc-auc": 0.6208476032754839, + "score": 0.6208476032754839 + }, + "test": { + "0": { + "precision": 0.6596955128205129, + "recall": 0.9730528306346767, + "f1-score": 0.7863043789694857, + "support": 16922.0 + }, + "1": { + "precision": 0.6949832775919732, + "recall": 0.10898982481904962, + "f1-score": 0.18842945230322813, + "support": 9533.0 + }, + "accuracy": 0.6616896616896617, + "macro avg": { + "precision": 0.677339395206243, + "recall": 0.5410213277268632, + "f1-score": 0.4873669156363569, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6724113798235871, + "recall": 0.6616896616896617, + "f1-score": 0.5708614881772184, + "support": 26455.0 + }, + "cross-entropy": 0.7527583302993925, + "roc-auc": 0.5890089208341323, + "score": 0.5890089208341323 + } + }, + "time": "0:01:18.718791", + "diversity": { + "train": { + "mean_centered_corr": -0.007062356453388929, + "mean_pairwise_disagreement": 0.025052193183149135, + "member_std": 0.028095407411456108 + }, + "val": { + "mean_centered_corr": -0.003141294466331601, + "mean_pairwise_disagreement": 0.09142868017301098, + "member_std": 0.06040506064891815 + }, + "test": { + "mean_centered_corr": 0.00333536509424448, + "mean_pairwise_disagreement": 0.05701769895318282, + "member_std": 0.059019479900598526 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5f7214ce5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,428 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7909659567005086, + "recall": 0.9643735373430428, + "f1-score": 0.8691043985161632, + "support": 84179.0 + }, + "1": { + "precision": 0.5528552258834054, + "recall": 0.14736507431841667, + "f1-score": 0.23270262637672973, + "support": 25162.0 + }, + "accuracy": 0.7763601942546712, + "macro avg": { + "precision": 0.671910591291957, + "recall": 0.5558693058307297, + "f1-score": 0.5509035124464464, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7361709373681451, + "recall": 0.7763601942546712, + "f1-score": 0.7226530089132472, + "support": 109341.0 + }, + "cross-entropy": 0.5000394688239905, + "roc-auc": 0.6809790060024956, + "score": 0.6809790060024956 + }, + "val": { + "0": { + "precision": 0.7075808427894092, + "recall": 0.8559092666580745, + "f1-score": 0.7747090903788387, + "support": 15518.0 + }, + "1": { + "precision": 0.5927140255009108, + "recall": 0.3721834610545579, + "f1-score": 0.4572472423241762, + "support": 8743.0 + }, + "accuracy": 0.6815877333992828, + "macro avg": { + "precision": 0.6501474341451601, + "recall": 0.6140463638563163, + "f1-score": 0.6159781663515075, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6661859875256797, + "recall": 0.6815877333992828, + "f1-score": 0.6603045341964096, + "support": 24261.0 + }, + "cross-entropy": 0.6203779736299148, + "roc-auc": 0.6506783465179156, + "score": 0.6506783465179156 + }, + "test": { + "0": { + "precision": 0.6800559465011583, + "recall": 0.9194539652523342, + "f1-score": 0.7818396522700435, + "support": 16922.0 + }, + "1": { + "precision": 0.618847874720358, + "recall": 0.23214098395048777, + "f1-score": 0.3376306354412999, + "support": 9533.0 + }, + "accuracy": 0.6717822717822718, + "macro avg": { + "precision": 0.6494519106107581, + "recall": 0.575797474601411, + "f1-score": 0.5597351438556717, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6579997549197419, + "recall": 0.6717822717822718, + "f1-score": 0.6217699279295251, + "support": 26455.0 + }, + "cross-entropy": 0.6443009499315193, + "roc-auc": 0.6065618416202598, + "score": 0.6065618416202598 + } + }, + "time": "0:01:18.654365", + "diversity": { + "train": { + "mean_centered_corr": -0.01063462533056736, + "mean_pairwise_disagreement": 0.026665487313881315, + "member_std": 0.029800426214933395 + }, + "val": { + "mean_centered_corr": -0.013281173072755337, + "mean_pairwise_disagreement": 0.09412748922670262, + "member_std": 0.06749773025512695 + }, + "test": { + "mean_centered_corr": -0.011601177044212818, + "mean_pairwise_disagreement": 0.059119563958273634, + "member_std": 0.06550896167755127 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 24, + 24 + ], + "head_selection_time": "0:00:20.435848" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..62045c384 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1/report.json @@ -0,0 +1,426 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9005681872367859, + "min_features_per_member": 89.0, + "mean_features_per_member": 99.0625, + "max_features_per_member": 106.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.81818199157715, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8186085273842617, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7888152627873106, + "recall": 0.9715249646586441, + "f1-score": 0.8706881866961929, + "support": 84179.0 + }, + "1": { + "precision": 0.5768008474576272, + "recall": 0.12983864557666322, + "f1-score": 0.21196392655550508, + "support": 25162.0 + }, + "accuracy": 0.7778326519786722, + "macro avg": { + "precision": 0.6828080551224689, + "recall": 0.5506818051176536, + "f1-score": 0.541326056625849, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.740025634756421, + "recall": 0.7778326519786722, + "f1-score": 0.7190998544725988, + "support": 109341.0 + }, + "cross-entropy": 0.497526106332481, + "roc-auc": 0.6852792162881653, + "score": 0.6852792162881653 + }, + "val": { + "0": { + "precision": 0.6946272468323348, + "recall": 0.9114576620698543, + "f1-score": 0.7884057971014493, + "support": 15518.0 + }, + "1": { + "precision": 0.6476019492177482, + "recall": 0.28880247054786684, + "f1-score": 0.39946211042556556, + "support": 8743.0 + }, + "accuracy": 0.6870697827789456, + "macro avg": { + "precision": 0.6711145980250415, + "recall": 0.6001300663088606, + "f1-score": 0.5939339537635074, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6776806173840709, + "recall": 0.6870697827789456, + "f1-score": 0.6482411438469564, + "support": 24261.0 + }, + "cross-entropy": 0.6172041286027814, + "roc-auc": 0.6512227217747169, + "score": 0.6512227217747169 + }, + "test": { + "0": { + "precision": 0.6737023929895517, + "recall": 0.9449828625457983, + "f1-score": 0.7866102612032072, + "support": 16922.0 + }, + "1": { + "precision": 0.6575947039352703, + "recall": 0.18755900555963495, + "f1-score": 0.29187071498530853, + "support": 9533.0 + }, + "accuracy": 0.6720468720468721, + "macro avg": { + "precision": 0.6656485484624111, + "recall": 0.5662709340527167, + "f1-score": 0.5392404880942578, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6678980233144632, + "recall": 0.6720468720468721, + "f1-score": 0.6083319737681201, + "support": 26455.0 + }, + "cross-entropy": 0.6427700166976323, + "roc-auc": 0.6075381341628896, + "score": 0.6075381341628896 + } + }, + "time": "0:01:23.222825", + "diversity": { + "train": { + "mean_centered_corr": -0.008371543139219284, + "mean_pairwise_disagreement": 0.02743249071932702, + "member_std": 0.027091743424534798 + }, + "val": { + "mean_centered_corr": 0.05517549440264702, + "mean_pairwise_disagreement": 0.07159946402762432, + "member_std": 0.054152194410562515 + }, + "test": { + "mean_centered_corr": 0.09162398427724838, + "mean_pairwise_disagreement": 0.04175463812560587, + "member_std": 0.052940964698791504 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:11.118299" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..704a44552 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2/report.json @@ -0,0 +1,427 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8940340876579285, + "min_features_per_member": 89.0, + "mean_features_per_member": 98.34375, + "max_features_per_member": 104.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.60909080505371, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8074834695307933, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.787158853016306, + "recall": 0.9737583007638485, + "f1-score": 0.870571867051133, + "support": 84179.0 + }, + "1": { + "precision": 0.5757633954292298, + "recall": 0.11914792146888165, + "f1-score": 0.19743817708847838, + "support": 25162.0 + }, + "accuracy": 0.7770918502665971, + "macro avg": { + "precision": 0.6814611242227679, + "recall": 0.5464531111163651, + "f1-score": 0.5340050220698057, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7385116620832981, + "recall": 0.7770918502665971, + "f1-score": 0.7156675776552036, + "support": 109341.0 + }, + "cross-entropy": 0.49884451202024715, + "roc-auc": 0.6825760332150292, + "score": 0.6825760332150292 + }, + "val": { + "0": { + "precision": 0.6974369447564587, + "recall": 0.8802680757829617, + "f1-score": 0.7782588878760255, + "support": 15518.0 + }, + "1": { + "precision": 0.6025668449197861, + "recall": 0.3222006176369667, + "f1-score": 0.41988373826203607, + "support": 8743.0 + }, + "accuracy": 0.6791558468323647, + "macro avg": { + "precision": 0.6500018948381223, + "recall": 0.6012343467099642, + "f1-score": 0.5990713130690308, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6632483588419446, + "recall": 0.6791558468323647, + "f1-score": 0.6491102982434831, + "support": 24261.0 + }, + "cross-entropy": 0.6255965554487835, + "roc-auc": 0.6436592869751769, + "score": 0.6436592869751769 + }, + "test": { + "0": { + "precision": 0.6723253228427264, + "recall": 0.9291454910767049, + "f1-score": 0.7801428996725216, + "support": 16922.0 + }, + "1": { + "precision": 0.6093189964157706, + "recall": 0.19616070491975243, + "f1-score": 0.29677828916045074, + "support": 9533.0 + }, + "accuracy": 0.665016065016065, + "macro avg": { + "precision": 0.6408221596292485, + "recall": 0.5626530979982286, + "f1-score": 0.5384605944164862, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6496211342270329, + "recall": 0.665016065016065, + "f1-score": 0.6059635448431293, + "support": 26455.0 + }, + "cross-entropy": 0.6535820200884032, + "roc-auc": 0.6015328994897303, + "score": 0.6015328994897303 + } + }, + "time": "0:01:18.718791", + "diversity": { + "train": { + "mean_centered_corr": -0.007062356453388929, + "mean_pairwise_disagreement": 0.025052193183149135, + "member_std": 0.028095407411456108 + }, + "val": { + "mean_centered_corr": -0.003141294466331601, + "mean_pairwise_disagreement": 0.09142868017301098, + "member_std": 0.06040506064891815 + }, + "test": { + "mean_centered_corr": 0.00333536509424448, + "mean_pairwise_disagreement": 0.05701769895318282, + "member_std": 0.059019479900598526 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 23, + 9 + ], + "head_selection_time": "0:00:15.951706" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..223176d4b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7841790931223371, + "recall": 0.9795554710794854, + "f1-score": 0.8710459459887709, + "support": 84179.0 + }, + "1": { + "precision": 0.5891620911912151, + "recall": 0.0980844130037358, + "f1-score": 0.16817144219958435, + "support": 25162.0 + }, + "accuracy": 0.7767077308603361, + "macro avg": { + "precision": 0.6866705921567762, + "recall": 0.5388199420416105, + "f1-score": 0.5196086940941776, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7393009796736684, + "recall": 0.7767077308603361, + "f1-score": 0.709297578365066, + "support": 109341.0 + }, + "cross-entropy": 0.4992703210863206, + "roc-auc": 0.6803857932728635, + "score": 0.6803857932728635 + }, + "val": { + "0": { + "precision": 0.688133775351014, + "recall": 0.9095888645444, + "f1-score": 0.7835137385512073, + "support": 15518.0 + }, + "1": { + "precision": 0.6257668711656442, + "recall": 0.26832894887338443, + "f1-score": 0.3756003842459174, + "support": 8743.0 + }, + "accuracy": 0.6784963521701496, + "macro avg": { + "precision": 0.6569503232583291, + "recall": 0.5889589067088923, + "f1-score": 0.5795570613985623, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6656584510324497, + "recall": 0.6784963521701496, + "f1-score": 0.63651293657721, + "support": 24261.0 + }, + "cross-entropy": 0.6310932933009203, + "roc-auc": 0.6410241296714208, + "score": 0.6410241296714208 + }, + "test": { + "0": { + "precision": 0.6670702684647822, + "recall": 0.9441555371705472, + "f1-score": 0.781787488072811, + "support": 16922.0 + }, + "1": { + "precision": 0.6226038338658147, + "recall": 0.16353718661491662, + "f1-score": 0.2590346431835175, + "support": 9533.0 + }, + "accuracy": 0.6628614628614629, + "macro avg": { + "precision": 0.6448370511652985, + "recall": 0.553846361892732, + "f1-score": 0.5204110656281642, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6510468883463941, + "recall": 0.6628614628614629, + "f1-score": 0.5934146712015339, + "support": 26455.0 + }, + "cross-entropy": 0.6635058051130894, + "roc-auc": 0.6024966050474919, + "score": 0.6024966050474919 + } + }, + "time": "0:01:33.915789", + "diversity": { + "train": { + "mean_centered_corr": -0.013111491687595844, + "mean_pairwise_disagreement": 0.02585577938919114, + "member_std": 0.02714134193956852 + }, + "val": { + "mean_centered_corr": -0.004636597353965044, + "mean_pairwise_disagreement": 0.08690653790565238, + "member_std": 0.060862503945827484 + }, + "test": { + "mean_centered_corr": 0.0027443149592727423, + "mean_pairwise_disagreement": 0.05465580626870949, + "member_std": 0.05927826836705208 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:04.756597" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..6ae6a0c85 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/1/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7921965289755508, + "recall": 0.9657515532377434, + "f1-score": 0.870406852248394, + "support": 84179.0 + }, + "1": { + "precision": 0.5709821428571429, + "recall": 0.15249185279389554, + "f1-score": 0.2407000815507183, + "support": 25162.0 + }, + "accuracy": 0.7786008907911945, + "macro avg": { + "precision": 0.6815893359163469, + "recall": 0.5591217030158195, + "f1-score": 0.5555534668995561, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7412897658810906, + "recall": 0.7786008907911945, + "f1-score": 0.7254961438746375, + "support": 109341.0 + }, + "cross-entropy": 0.49324286572021286, + "roc-auc": 0.6956759811999328, + "score": 0.6956759811999328 + }, + "val": { + "0": { + "precision": 0.6824444652732214, + "recall": 0.9383941229539889, + "f1-score": 0.7902105491643152, + "support": 15518.0 + }, + "1": { + "precision": 0.6729387615463565, + "recall": 0.22497998398718974, + "f1-score": 0.3372192696725527, + "support": 8743.0 + }, + "accuracy": 0.6812992044845637, + "macro avg": { + "precision": 0.677691613409789, + "recall": 0.5816870534705894, + "f1-score": 0.5637149094184339, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6790188699686593, + "recall": 0.6812992044845637, + "f1-score": 0.6269648974353478, + "support": 24261.0 + }, + "cross-entropy": 0.6631863406542986, + "roc-auc": 0.6315182980623079, + "score": 0.6315182980623079 + }, + "test": { + "0": { + "precision": 0.6619854721549637, + "recall": 0.9693889611157074, + "f1-score": 0.7867248573209918, + "support": 16922.0 + }, + "1": { + "precision": 0.6907462686567164, + "recall": 0.12136787999580405, + "f1-score": 0.20645967166309778, + "support": 9533.0 + }, + "accuracy": 0.6638064638064638, + "macro avg": { + "precision": 0.6763658704058401, + "recall": 0.5453784205557557, + "f1-score": 0.4965922644920448, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.672349360760188, + "recall": 0.6638064638064638, + "f1-score": 0.5776275972613923, + "support": 26455.0 + }, + "cross-entropy": 0.7198854018171156, + "roc-auc": 0.5962485571769538, + "score": 0.5962485571769538 + } + }, + "time": "0:01:39.396326", + "diversity": { + "train": { + "mean_centered_corr": -0.00980177242308855, + "mean_pairwise_disagreement": 0.02483125814446725, + "member_std": 0.02443780191242695 + }, + "val": { + "mean_centered_corr": 0.02159964293241501, + "mean_pairwise_disagreement": 0.06877450667538902, + "member_std": 0.04672974348068237 + }, + "test": { + "mean_centered_corr": 0.04072229564189911, + "mean_pairwise_disagreement": 0.03965246828150054, + "member_std": 0.04293917864561081 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 15 + ], + "head_selection_time": "0:00:04.804613" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..b7ffa6124 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-best-head-evaluation/2/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7891122957469, + "recall": 0.9706815238955084, + "f1-score": 0.8705300782519296, + "support": 84179.0 + }, + "1": { + "precision": 0.5739685827723114, + "recall": 0.13214370876718862, + "f1-score": 0.2148279760943305, + "support": 25162.0 + }, + "accuracy": 0.7777137578767342, + "macro avg": { + "precision": 0.6815404392596057, + "recall": 0.5514126163313485, + "f1-score": 0.5426790271731301, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7396025408894668, + "recall": 0.7777137578767342, + "f1-score": 0.7196372174358633, + "support": 109341.0 + }, + "cross-entropy": 0.4988365072053171, + "roc-auc": 0.6810069006558737, + "score": 0.6810069006558737 + }, + "val": { + "0": { + "precision": 0.6966991903674487, + "recall": 0.8650599304034025, + "f1-score": 0.7718047490369689, + "support": 15518.0 + }, + "1": { + "precision": 0.5806128580012017, + "recall": 0.3315795493537687, + "f1-score": 0.4221025043680839, + "support": 8743.0 + }, + "accuracy": 0.6728082107085446, + "macro avg": { + "precision": 0.6386560241843251, + "recall": 0.5983197398785856, + "f1-score": 0.5969536267025264, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6548648552667481, + "recall": 0.6728082107085446, + "f1-score": 0.6457816368346664, + "support": 24261.0 + }, + "cross-entropy": 0.6306857322286159, + "roc-auc": 0.6341914987995403, + "score": 0.6341914987995403 + }, + "test": { + "0": { + "precision": 0.6714620131942967, + "recall": 0.9322775085687272, + "f1-score": 0.7806616027909048, + "support": 16922.0 + }, + "1": { + "precision": 0.6128378378378379, + "recall": 0.1902863736494283, + "f1-score": 0.29040262547026335, + "support": 9533.0 + }, + "accuracy": 0.6649026649026649, + "macro avg": { + "precision": 0.6421499255160672, + "recall": 0.5612819411090777, + "f1-score": 0.5355321141305841, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6503369229023624, + "recall": 0.6649026649026649, + "f1-score": 0.6039978783230282, + "support": 26455.0 + }, + "cross-entropy": 0.6517993050987927, + "roc-auc": 0.599267480873393, + "score": 0.599267480873393 + } + }, + "time": "0:01:33.653112", + "diversity": { + "train": { + "mean_centered_corr": -0.015336746349930763, + "mean_pairwise_disagreement": 0.023258879958555228, + "member_std": 0.02414415217936039 + }, + "val": { + "mean_centered_corr": -0.00032689498038962483, + "mean_pairwise_disagreement": 0.07033457387470399, + "member_std": 0.050614725798368454 + }, + "test": { + "mean_centered_corr": 0.007613664958626032, + "mean_pairwise_disagreement": 0.041485389066034224, + "member_std": 0.04775906354188919 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:04.781830" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml new file mode 100644 index 000000000..9b75584bc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/0/report.json new file mode 100644 index 000000000..517e309dc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7862810438170169, + "recall": 0.9782487318689934, + "f1-score": 0.8718225610078874, + "support": 84179.0 + }, + "1": { + "precision": 0.602819956616052, + "recall": 0.11044432080120817, + "f1-score": 0.18668547628644364, + "support": 25162.0 + }, + "accuracy": 0.7785460165903001, + "macro avg": { + "precision": 0.6945505002165344, + "recall": 0.5443465263351008, + "f1-score": 0.5292540186471655, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7440622249279388, + "recall": 0.7785460165903001, + "f1-score": 0.7141560011103103, + "support": 109341.0 + }, + "cross-entropy": 0.4922902286245573, + "roc-auc": 0.6989728271677539, + "score": 0.6989728271677539 + }, + "val": { + "0": { + "precision": 0.675189954047045, + "recall": 0.9563088026807578, + "f1-score": 0.791529989065792, + "support": 15518.0 + }, + "1": { + "precision": 0.7028921998247152, + "recall": 0.18346105455793205, + "f1-score": 0.2909750566893424, + "support": 8743.0 + }, + "accuracy": 0.677795639091546, + "macro avg": { + "precision": 0.6890410769358801, + "recall": 0.5698849286193449, + "f1-score": 0.5412525228775672, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6851730847850266, + "recall": 0.677795639091546, + "f1-score": 0.6111436993923531, + "support": 24261.0 + }, + "cross-entropy": 0.6872600691770208, + "roc-auc": 0.6227874903903754, + "score": 0.6227874903903754 + }, + "test": { + "0": { + "precision": 0.656414727914074, + "recall": 0.9787259189221132, + "f1-score": 0.7858040946077385, + "support": 16922.0 + }, + "1": { + "precision": 0.7058823529411765, + "recall": 0.09063253959928669, + "f1-score": 0.16063958352700566, + "support": 9533.0 + }, + "accuracy": 0.6587034587034587, + "macro avg": { + "precision": 0.6811485404276252, + "recall": 0.5346792292607, + "f1-score": 0.4732218390673721, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6742402758023133, + "recall": 0.6587034587034587, + "f1-score": 0.5605274631908937, + "support": 26455.0 + }, + "cross-entropy": 0.7564828184565525, + "roc-auc": 0.5911993816464689, + "score": 0.5911993816464689 + } + }, + "time": "0:01:33.915789", + "diversity": { + "train": { + "mean_centered_corr": -0.013111491687595844, + "mean_pairwise_disagreement": 0.02585577938919114, + "member_std": 0.02714134193956852 + }, + "val": { + "mean_centered_corr": -0.004636597353965044, + "mean_pairwise_disagreement": 0.08690653790565238, + "member_std": 0.060862503945827484 + }, + "test": { + "mean_centered_corr": 0.0027443149592727423, + "mean_pairwise_disagreement": 0.05465580626870949, + "member_std": 0.05927826836705208 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml new file mode 100644 index 000000000..b80a19935 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/1/report.json new file mode 100644 index 000000000..d79e09c49 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/1/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7880230326295585, + "recall": 0.9754451822901199, + "f1-score": 0.8717744546897478, + "support": 84179.0 + }, + "1": { + "precision": 0.5979381443298969, + "recall": 0.12216834909784596, + "f1-score": 0.2028842028842029, + "support": 25162.0 + }, + "accuracy": 0.7790856128990955, + "macro avg": { + "precision": 0.6929805884797278, + "recall": 0.5488067656939829, + "f1-score": 0.5373293287869754, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.744279917426697, + "recall": 0.7790856128990955, + "f1-score": 0.7178466827109738, + "support": 109341.0 + }, + "cross-entropy": 0.49036011907714316, + "roc-auc": 0.7032037382378304, + "score": 0.7032037382378304 + }, + "val": { + "0": { + "precision": 0.6734879077998094, + "recall": 0.9565021265627014, + "f1-score": 0.7904252203317624, + "support": 15518.0 + }, + "1": { + "precision": 0.6962196219621962, + "recall": 0.17694155324259409, + "f1-score": 0.28217054263565894, + "support": 8743.0 + }, + "accuracy": 0.6755698446065702, + "macro avg": { + "precision": 0.6848537648810028, + "recall": 0.5667218399026477, + "f1-score": 0.5362978814837107, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6816797950642152, + "recall": 0.6755698446065702, + "f1-score": 0.6072641533066179, + "support": 24261.0 + }, + "cross-entropy": 0.7136592088387013, + "roc-auc": 0.6204141557865444, + "score": 0.6204141557865444 + }, + "test": { + "0": { + "precision": 0.6556994100645366, + "recall": 0.9786668242524524, + "f1-score": 0.7852722918988122, + "support": 16922.0 + }, + "1": { + "precision": 0.6986644407345576, + "recall": 0.08780027273680897, + "f1-score": 0.15599664523343584, + "support": 9533.0 + }, + "accuracy": 0.6576450576450577, + "macro avg": { + "precision": 0.677181925399547, + "recall": 0.5332335484946307, + "f1-score": 0.470634468566124, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6711817626397515, + "recall": 0.6576450576450577, + "f1-score": 0.5585142219815552, + "support": 26455.0 + }, + "cross-entropy": 0.7976474685558286, + "roc-auc": 0.5897760078319128, + "score": 0.5897760078319128 + } + }, + "time": "0:01:39.396326", + "diversity": { + "train": { + "mean_centered_corr": -0.00980177242308855, + "mean_pairwise_disagreement": 0.02483125814446725, + "member_std": 0.02443780191242695 + }, + "val": { + "mean_centered_corr": 0.02159964293241501, + "mean_pairwise_disagreement": 0.06877450667538902, + "member_std": 0.04672974348068237 + }, + "test": { + "mean_centered_corr": 0.04072229564189911, + "mean_pairwise_disagreement": 0.03965246828150054, + "member_std": 0.04293917864561081 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml new file mode 100644 index 000000000..029b6e363 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/2/report.json new file mode 100644 index 000000000..ab12d4221 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/2/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7859541664281489, + "recall": 0.9786169947374048, + "f1-score": 0.8717677374294286, + "support": 84179.0 + }, + "1": { + "precision": 0.6023856858846919, + "recall": 0.10837771242349575, + "f1-score": 0.1837044023038836, + "support": 25162.0 + }, + "accuracy": 0.7783539568871695, + "macro avg": { + "precision": 0.6941699261564204, + "recall": 0.5434973535804503, + "f1-score": 0.5277360698666561, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7437106337420158, + "recall": 0.7783539568871695, + "f1-score": 0.7134277767703074, + "support": 109341.0 + }, + "cross-entropy": 0.4919189215910807, + "roc-auc": 0.6999636786911775, + "score": 0.6999636786911775 + }, + "val": { + "0": { + "precision": 0.6763377583139455, + "recall": 0.9554066245650212, + "f1-score": 0.7920083335559176, + "support": 15518.0 + }, + "1": { + "precision": 0.7042735042735043, + "recall": 0.18849365206450874, + "f1-score": 0.29739240277903095, + "support": 8743.0 + }, + "accuracy": 0.6790321915831994, + "macro avg": { + "precision": 0.6903056312937249, + "recall": 0.5719501383147649, + "f1-score": 0.5447003681674742, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6864050361229568, + "recall": 0.6790321915831994, + "f1-score": 0.6137622974163388, + "support": 24261.0 + }, + "cross-entropy": 0.7025996858462662, + "roc-auc": 0.6211550537725488, + "score": 0.6211550537725488 + }, + "test": { + "0": { + "precision": 0.6576476205634251, + "recall": 0.9767167001536461, + "f1-score": 0.7860370000475579, + "support": 16922.0 + }, + "1": { + "precision": 0.7021919879062736, + "recall": 0.09745095982377006, + "f1-score": 0.17114959469417834, + "support": 9533.0 + }, + "accuracy": 0.6598752598752599, + "macro avg": { + "precision": 0.6799198042348493, + "recall": 0.5370838299887081, + "f1-score": 0.47859329737086814, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6736990835715285, + "recall": 0.6598752598752599, + "f1-score": 0.5644637006624221, + "support": 26455.0 + }, + "cross-entropy": 0.7802766696915584, + "roc-auc": 0.5892400551940371, + "score": 0.5892400551940371 + } + }, + "time": "0:01:33.653112", + "diversity": { + "train": { + "mean_centered_corr": -0.015336746349930763, + "mean_pairwise_disagreement": 0.023258879958555228, + "member_std": 0.02414415217936039 + }, + "val": { + "mean_centered_corr": -0.00032689498038962483, + "mean_pairwise_disagreement": 0.07033457387470399, + "member_std": 0.050614725798368454 + }, + "test": { + "mean_centered_corr": 0.007613664958626032, + "mean_pairwise_disagreement": 0.041485389066034224, + "member_std": 0.04775906354188919 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..a92177d15 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,230 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.782973008757495, + "recall": 0.9834994476056974, + "f1-score": 0.871854546986315, + "support": 84179.0 + }, + "1": { + "precision": 0.6144879267277269, + "recall": 0.08798982592798664, + "f1-score": 0.1539370763080132, + "support": 25162.0 + }, + "accuracy": 0.7774210954719638, + "macro avg": { + "precision": 0.6987304677426109, + "recall": 0.535744636766842, + "f1-score": 0.5128958116471641, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7442005296871277, + "recall": 0.7774210954719638, + "f1-score": 0.7066444300383502, + "support": 109341.0 + }, + "cross-entropy": 0.4979289521091686, + "roc-auc": 0.6847935169951291, + "score": 0.6847935169951291 + }, + "val": { + "0": { + "precision": 0.6776083871263104, + "recall": 0.9537955922154917, + "f1-score": 0.7923235459436311, + "support": 15518.0 + }, + "1": { + "precision": 0.7034739454094293, + "recall": 0.1945556445156125, + "f1-score": 0.3048113968282412, + "support": 8743.0 + }, + "accuracy": 0.6801863072420757, + "macro avg": { + "precision": 0.6905411662678699, + "recall": 0.5741756183655521, + "f1-score": 0.5485674713859361, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6869296259898902, + "recall": 0.6801863072420757, + "f1-score": 0.6166375181741305, + "support": 24261.0 + }, + "cross-entropy": 0.639843807695552, + "roc-auc": 0.641845068859757, + "score": 0.641845068859757 + }, + "test": { + "0": { + "precision": 0.6588995730758489, + "recall": 0.975889374778395, + "f1-score": 0.7866619030606169, + "support": 16922.0 + }, + "1": { + "precision": 0.7068965517241379, + "recall": 0.10322039232140984, + "f1-score": 0.18013729977116705, + "support": 9533.0 + }, + "accuracy": 0.6614250614250614, + "macro avg": { + "precision": 0.6828980623999934, + "recall": 0.5395548835499024, + "f1-score": 0.483399601415892, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6761951768352191, + "recall": 0.6614250614250614, + "f1-score": 0.5681021206694499, + "support": 26455.0 + }, + "cross-entropy": 0.6818687369091303, + "roc-auc": 0.6041622806453656, + "score": 0.6041622806453656 + } + }, + "time": "0:01:33.915789", + "diversity": { + "train": { + "mean_centered_corr": -0.013111491687595844, + "mean_pairwise_disagreement": 0.02585577938919114, + "member_std": 0.02714134193956852 + }, + "val": { + "mean_centered_corr": -0.004636597353965044, + "mean_pairwise_disagreement": 0.08690653790565238, + "member_std": 0.060862503945827484 + }, + "test": { + "mean_centered_corr": 0.0027443149592727423, + "mean_pairwise_disagreement": 0.05465580626870949, + "member_std": 0.05927826836705208 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 25 + ], + "head_selection_time": "0:00:15.827629" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..4e16edaf9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,233 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7931243640917274, + "recall": 0.9630667981325509, + "f1-score": 0.869873118914134, + "support": 84179.0 + }, + "1": { + "precision": 0.5636491228070175, + "recall": 0.15960575470948254, + "f1-score": 0.2487688543376591, + "support": 25162.0 + }, + "accuracy": 0.778171042884188, + "macro avg": { + "precision": 0.6783867434493724, + "recall": 0.5613362764210167, + "f1-score": 0.5593209866258966, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7403165790778179, + "recall": 0.778171042884188, + "f1-score": 0.7269420545807801, + "support": 109341.0 + }, + "cross-entropy": 0.4937839932106913, + "roc-auc": 0.6959329827185088, + "score": 0.6959329827185088 + }, + "val": { + "0": { + "precision": 0.6899277108433735, + "recall": 0.9225415646346179, + "f1-score": 0.7894562699900739, + "support": 15518.0 + }, + "1": { + "precision": 0.6576473939048704, + "recall": 0.26409699187921765, + "f1-score": 0.3768565366410968, + "support": 8743.0 + }, + "accuracy": 0.6852561724578542, + "macro avg": { + "precision": 0.6737875523741219, + "recall": 0.5933192782569178, + "f1-score": 0.5831564033155854, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6782947686318681, + "recall": 0.6852561724578542, + "f1-score": 0.6407666253476393, + "support": 24261.0 + }, + "cross-entropy": 0.6548096037426314, + "roc-auc": 0.6326258289049813, + "score": 0.6326258289049813 + }, + "test": { + "0": { + "precision": 0.6649503493931592, + "recall": 0.9615884647204822, + "f1-score": 0.7862198922523131, + "support": 16922.0 + }, + "1": { + "precision": 0.6723790322580645, + "recall": 0.1399349627609357, + "f1-score": 0.23165754970912564, + "support": 9533.0 + }, + "accuracy": 0.6655074655074655, + "macro avg": { + "precision": 0.6686646908256119, + "recall": 0.5507617137407089, + "f1-score": 0.5089387209807194, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6676272586258615, + "recall": 0.6655074655074655, + "f1-score": 0.5863845941436681, + "support": 26455.0 + }, + "cross-entropy": 0.7047260741581922, + "roc-auc": 0.5972452690882881, + "score": 0.5972452690882881 + } + }, + "time": "0:01:39.396326", + "diversity": { + "train": { + "mean_centered_corr": -0.00980177242308855, + "mean_pairwise_disagreement": 0.02483125814446725, + "member_std": 0.02443780191242695 + }, + "val": { + "mean_centered_corr": 0.02159964293241501, + "mean_pairwise_disagreement": 0.06877450667538902, + "member_std": 0.04672974348068237 + }, + "test": { + "mean_centered_corr": 0.04072229564189911, + "mean_pairwise_disagreement": 0.03965246828150054, + "member_std": 0.04293917864561081 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 15, + 10, + 9, + 20, + 9 + ], + "head_selection_time": "0:00:29.881915" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..d7841fe53 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,230 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7868832039858197, + "recall": 0.9756233740006415, + "f1-score": 0.8711475531559435, + "support": 84179.0 + }, + "1": { + "precision": 0.5872057936028968, + "recall": 0.11600826643351085, + "f1-score": 0.1937410812066505, + "support": 25162.0 + }, + "accuracy": 0.777805214878225, + "macro avg": { + "precision": 0.6870444987943582, + "recall": 0.5458158202170762, + "f1-score": 0.532444317181297, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7409326182032212, + "recall": 0.777805214878225, + "f1-score": 0.7152599936202879, + "support": 109341.0 + }, + "cross-entropy": 0.49847322238722597, + "roc-auc": 0.682215318814317, + "score": 0.682215318814317 + }, + "val": { + "0": { + "precision": 0.6895468820098699, + "recall": 0.8914164196417064, + "f1-score": 0.7775935242699346, + "support": 15518.0 + }, + "1": { + "precision": 0.5988095238095238, + "recall": 0.28765869838728125, + "f1-score": 0.38862705709650003, + "support": 8743.0 + }, + "accuracy": 0.6738386711182557, + "macro avg": { + "precision": 0.6441782029096969, + "recall": 0.5895375590144938, + "f1-score": 0.5831102906832173, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6568476230038262, + "recall": 0.6738386711182557, + "f1-score": 0.63742066154798, + "support": 24261.0 + }, + "cross-entropy": 0.6333810186447947, + "roc-auc": 0.6354981541987959, + "score": 0.6354981541987959 + }, + "test": { + "0": { + "precision": 0.6670303844216887, + "recall": 0.9392506795887011, + "f1-score": 0.7800736196319018, + "support": 16922.0 + }, + "1": { + "precision": 0.6086791016368481, + "recall": 0.167733137522291, + "f1-score": 0.26299342105263157, + "support": 9533.0 + }, + "accuracy": 0.6612360612360613, + "macro avg": { + "precision": 0.6378547430292685, + "recall": 0.553491908555496, + "f1-score": 0.5215335203422666, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6460036303567527, + "recall": 0.6612360612360613, + "f1-score": 0.5937449281536865, + "support": 26455.0 + }, + "cross-entropy": 0.6626198047400376, + "roc-auc": 0.5998512677731419, + "score": 0.5998512677731419 + } + }, + "time": "0:01:33.653112", + "diversity": { + "train": { + "mean_centered_corr": -0.015336746349930763, + "mean_pairwise_disagreement": 0.023258879958555228, + "member_std": 0.02414415217936039 + }, + "val": { + "mean_centered_corr": -0.00032689498038962483, + "mean_pairwise_disagreement": 0.07033457387470399, + "member_std": 0.050614725798368454 + }, + "test": { + "mean_centered_corr": 0.007613664958626032, + "mean_pairwise_disagreement": 0.041485389066034224, + "member_std": 0.04775906354188919 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 20 + ], + "head_selection_time": "0:00:15.768462" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..85c7d6ec0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0/report.json @@ -0,0 +1,255 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.6894886493682861, + "min_features_per_member": 67.0, + "mean_features_per_member": 75.84375, + "max_features_per_member": 84.0, + "min_members_per_feature": 14.0, + "mean_members_per_feature": 22.063636779785156, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.5278457608823273, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7894966160281175, + "recall": 0.9686501383955618, + "f1-score": 0.8699455883921904, + "support": 84179.0 + }, + "1": { + "precision": 0.5645214521452145, + "recall": 0.13595898577219617, + "f1-score": 0.21914034975337904, + "support": 25162.0 + }, + "accuracy": 0.7770278303655537, + "macro avg": { + "precision": 0.677009034086666, + "recall": 0.552304562083879, + "f1-score": 0.5445429690727848, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7377244073084094, + "recall": 0.7770278303655537, + "f1-score": 0.7201796139212255, + "support": 109341.0 + }, + "cross-entropy": 0.5002029863194409, + "roc-auc": 0.6821879786641952, + "score": 0.6821879786641952 + }, + "val": { + "0": { + "precision": 0.6961402811584124, + "recall": 0.9030802938523006, + "f1-score": 0.7862212123762238, + "support": 15518.0 + }, + "1": { + "precision": 0.6358353510895883, + "recall": 0.30035456936978155, + "f1-score": 0.4079857065175173, + "support": 8743.0 + }, + "accuracy": 0.6858744487036808, + "macro avg": { + "precision": 0.6659878161240004, + "recall": 0.6017174316110411, + "f1-score": 0.5971034594468705, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6744080358432264, + "recall": 0.6858744487036808, + "f1-score": 0.6499154942391862, + "support": 24261.0 + }, + "cross-entropy": 0.6154470541736884, + "roc-auc": 0.6547032702847417, + "score": 0.6547032702847417 + }, + "test": { + "0": { + "precision": 0.6733296707940667, + "recall": 0.9415553717054722, + "f1-score": 0.7851669335961562, + "support": 16922.0 + }, + "1": { + "precision": 0.6457736389684814, + "recall": 0.18913248714990036, + "f1-score": 0.2925760649087221, + "support": 9533.0 + }, + "accuracy": 0.6704214704214704, + "macro avg": { + "precision": 0.6595516548812741, + "recall": 0.5653439294276863, + "f1-score": 0.5388714992524392, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6633999164416454, + "recall": 0.6704214704214704, + "f1-score": 0.6076629172968816, + "support": 26455.0 + }, + "cross-entropy": 0.638677137478482, + "roc-auc": 0.6118564029158264, + "score": 0.6118564029158264 + } + }, + "time": "0:01:10.628940", + "diversity": { + "train": { + "mean_centered_corr": -0.015634549781680107, + "mean_pairwise_disagreement": 0.03130809179096706, + "member_std": 0.0355970673263073 + }, + "val": { + "mean_centered_corr": -0.0187542624771595, + "mean_pairwise_disagreement": 0.12205770312369114, + "member_std": 0.07598911970853806 + }, + "test": { + "mean_centered_corr": -0.018549803644418716, + "mean_pairwise_disagreement": 0.08149855811146134, + "member_std": 0.0770004391670227 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:04.834038" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..5f56a5253 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1/report.json @@ -0,0 +1,255 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.703125, + "min_features_per_member": 67.0, + "mean_features_per_member": 77.34375, + "max_features_per_member": 88.0, + "min_members_per_feature": 14.0, + "mean_members_per_feature": 22.5, + "max_members_per_feature": 29.0, + "mean_pairwise_jaccard": 0.5422206532495487, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7894095930260653, + "recall": 0.9703251404744652, + "f1-score": 0.8705675459632294, + "support": 84179.0 + }, + "1": { + "precision": 0.5744463373083475, + "recall": 0.13401160480089022, + "f1-score": 0.21732405259087392, + "support": 25162.0 + }, + "accuracy": 0.7778692347792685, + "macro avg": { + "precision": 0.6819279651672063, + "recall": 0.5521683726376777, + "f1-score": 0.5439457992770517, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7399413657337485, + "recall": 0.7778692347792685, + "f1-score": 0.7202404702987009, + "support": 109341.0 + }, + "cross-entropy": 0.4976395217774722, + "roc-auc": 0.6850579661840904, + "score": 0.6850579661840904 + }, + "val": { + "0": { + "precision": 0.7023705004389815, + "recall": 0.8764015981440907, + "f1-score": 0.779794157277601, + "support": 15518.0 + }, + "1": { + "precision": 0.6084115965700286, + "recall": 0.3408441038545122, + "f1-score": 0.43691811450773405, + "support": 8743.0 + }, + "accuracy": 0.6834013437203743, + "macro avg": { + "precision": 0.6553910485045051, + "recall": 0.6086228509993015, + "f1-score": 0.6083561358926676, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6685102845976619, + "recall": 0.6834013437203743, + "f1-score": 0.6562310213006443, + "support": 24261.0 + }, + "cross-entropy": 0.617130934303606, + "roc-auc": 0.6525683493050402, + "score": 0.6525683493050402 + }, + "test": { + "0": { + "precision": 0.678315295344895, + "recall": 0.9222314147263917, + "f1-score": 0.7816874953041649, + "support": 16922.0 + }, + "1": { + "precision": 0.6183294663573086, + "recall": 0.22364418336305467, + "f1-score": 0.3284800862799476, + "support": 9533.0 + }, + "accuracy": 0.6704970704970705, + "macro avg": { + "precision": 0.6483223808511018, + "recall": 0.5729377990447232, + "f1-score": 0.5550837907920563, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6566995362166145, + "recall": 0.6704970704970705, + "f1-score": 0.6183752204892768, + "support": 26455.0 + }, + "cross-entropy": 0.6387111341102191, + "roc-auc": 0.6078370975247275, + "score": 0.6078370975247275 + } + }, + "time": "0:01:12.475731", + "diversity": { + "train": { + "mean_centered_corr": -0.013842575252056122, + "mean_pairwise_disagreement": 0.028755187603386977, + "member_std": 0.03266480937600136 + }, + "val": { + "mean_centered_corr": -0.0060155028477311134, + "mean_pairwise_disagreement": 0.08212387197825795, + "member_std": 0.06719459593296051 + }, + "test": { + "mean_centered_corr": 0.0020387060940265656, + "mean_pairwise_disagreement": 0.052944440041214236, + "member_std": 0.0697987824678421 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:04.894692" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..e19938a28 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2/report.json @@ -0,0 +1,255 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.7090908885002136, + "min_features_per_member": 65.0, + "mean_features_per_member": 78.0, + "max_features_per_member": 92.0, + "min_members_per_feature": 17.0, + "mean_members_per_feature": 22.690908432006836, + "max_members_per_feature": 28.0, + "mean_pairwise_jaccard": 0.5463052047602153, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7825240884186662, + "recall": 0.9840696610793666, + "f1-score": 0.8717999989475844, + "support": 84179.0 + }, + "1": { + "precision": 0.6147658718758977, + "recall": 0.08504888323662665, + "f1-score": 0.14942568864993191, + "support": 25162.0 + }, + "accuracy": 0.7771833072680879, + "macro avg": { + "precision": 0.6986449801472819, + "recall": 0.5345592721579966, + "f1-score": 0.5106128437987582, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439188786195136, + "recall": 0.7771833072680879, + "f1-score": 0.7055642557615013, + "support": 109341.0 + }, + "cross-entropy": 0.49927567921826405, + "roc-auc": 0.6841046410049181, + "score": 0.6841046410049181 + }, + "val": { + "0": { + "precision": 0.6951219512195121, + "recall": 0.9256347467457147, + "f1-score": 0.7939859598695483, + "support": 15518.0 + }, + "1": { + "precision": 0.6791770920211287, + "recall": 0.27942353883106485, + "f1-score": 0.3959481361426256, + "support": 8743.0 + }, + "accuracy": 0.6927579242405507, + "macro avg": { + "precision": 0.6871495216203205, + "recall": 0.6025291427883898, + "f1-score": 0.5949670480060869, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.689375860622609, + "recall": 0.6927579242405507, + "f1-score": 0.6505440286694953, + "support": 24261.0 + }, + "cross-entropy": 0.6149192758236253, + "roc-auc": 0.6546592087434608, + "score": 0.6546592087434608 + }, + "test": { + "0": { + "precision": 0.6737775546910947, + "recall": 0.9518969388961116, + "f1-score": 0.7890470009062186, + "support": 16922.0 + }, + "1": { + "precision": 0.6805337519623234, + "recall": 0.18189447183467952, + "f1-score": 0.2870623292773777, + "support": 9533.0 + }, + "accuracy": 0.6744282744282745, + "macro avg": { + "precision": 0.677155653326709, + "recall": 0.5668957053653956, + "f1-score": 0.5380546650917981, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6762121352462495, + "recall": 0.6744282744282745, + "f1-score": 0.6081579487558598, + "support": 26455.0 + }, + "cross-entropy": 0.6464507795618373, + "roc-auc": 0.6088559366177836, + "score": 0.6088559366177836 + } + }, + "time": "0:01:07.040718", + "diversity": { + "train": { + "mean_centered_corr": -0.007606011815369129, + "mean_pairwise_disagreement": 0.026396666421797922, + "member_std": 0.03116724081337452 + }, + "val": { + "mean_centered_corr": 0.0018892597872763872, + "mean_pairwise_disagreement": 0.09870647301988722, + "member_std": 0.06490844488143921 + }, + "test": { + "mean_centered_corr": 0.01152767427265644, + "mean_pairwise_disagreement": 0.06456551295260973, + "member_std": 0.06369828432798386 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:04.886969" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml new file mode 100644 index 000000000..5319a963b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.7 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..d3d96f1bb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0/report.json @@ -0,0 +1,251 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.6894886493682861, + "min_features_per_member": 67.0, + "mean_features_per_member": 75.84375, + "max_features_per_member": 84.0, + "min_members_per_feature": 14.0, + "mean_members_per_feature": 22.063636779785156, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.5278457608823273, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7830881310769057, + "recall": 0.9845092006319865, + "f1-score": 0.8723225093416136, + "support": 84179.0 + }, + "1": { + "precision": 0.6284900284900284, + "recall": 0.08767188617756935, + "f1-score": 0.15387834821428573, + "support": 25162.0 + }, + "accuracy": 0.7781253143834427, + "macro avg": { + "precision": 0.705789079783467, + "recall": 0.5360905434047779, + "f1-score": 0.5131004287779497, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7475113807518583, + "recall": 0.7781253143834427, + "f1-score": 0.7069911882243216, + "support": 109341.0 + }, + "cross-entropy": 0.4934543091460455, + "roc-auc": 0.6961052398986505, + "score": 0.6961052398986505 + }, + "val": { + "0": { + "precision": 0.6825136099117702, + "recall": 0.9371697383683464, + "f1-score": 0.7898224080812469, + "support": 15518.0 + }, + "1": { + "precision": 0.6698272942770064, + "recall": 0.22623813336383392, + "f1-score": 0.3382352941176471, + "support": 8743.0 + }, + "accuracy": 0.6809694571534561, + "macro avg": { + "precision": 0.6761704520943883, + "recall": 0.5817039358660901, + "f1-score": 0.564028851099447, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6779418091782992, + "recall": 0.6809694571534561, + "f1-score": 0.627082779154832, + "support": 24261.0 + }, + "cross-entropy": 0.6467973900252932, + "roc-auc": 0.6291199291618959, + "score": 0.6291199291618959 + }, + "test": { + "0": { + "precision": 0.661515493867011, + "recall": 0.9688571090887602, + "f1-score": 0.7862178103869947, + "support": 16922.0 + }, + "1": { + "precision": 0.6846199880311191, + "recall": 0.12000419595090737, + "f1-score": 0.20421278114958943, + "support": 9533.0 + }, + "accuracy": 0.662974862974863, + "macro avg": { + "precision": 0.6730677409490651, + "recall": 0.5444306525198338, + "f1-score": 0.49521529576829204, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6698411465930153, + "recall": 0.662974862974863, + "f1-score": 0.5764936015901629, + "support": 26455.0 + }, + "cross-entropy": 0.6863610680565896, + "roc-auc": 0.5961977102213372, + "score": 0.5961977102213372 + } + }, + "time": "0:01:10.628940", + "diversity": { + "train": { + "mean_centered_corr": -0.015634549781680107, + "mean_pairwise_disagreement": 0.03130809179096706, + "member_std": 0.0355970673263073 + }, + "val": { + "mean_centered_corr": -0.0187542624771595, + "mean_pairwise_disagreement": 0.12205770312369114, + "member_std": 0.07598911970853806 + }, + "test": { + "mean_centered_corr": -0.018549803644418716, + "mean_pairwise_disagreement": 0.08149855811146134, + "member_std": 0.0770004391670227 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml new file mode 100644 index 000000000..503e15153 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.7 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 1 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1/report.json new file mode 100644 index 000000000..aef442869 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1/report.json @@ -0,0 +1,251 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.703125, + "min_features_per_member": 67.0, + "mean_features_per_member": 77.34375, + "max_features_per_member": 88.0, + "min_members_per_feature": 14.0, + "mean_members_per_feature": 22.5, + "max_members_per_feature": 29.0, + "mean_pairwise_jaccard": 0.5422206532495487, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7825586551070273, + "recall": 0.9854239180793309, + "f1-score": 0.8723525081501735, + "support": 84179.0 + }, + "1": { + "precision": 0.6326347305389222, + "recall": 0.08397583657896829, + "f1-score": 0.1482702968212757, + "support": 25162.0 + }, + "accuracy": 0.7779789831810574, + "macro avg": { + "precision": 0.7075966928229747, + "recall": 0.5346998773291496, + "f1-score": 0.5103114024857246, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7480575458252148, + "recall": 0.7779789831810574, + "f1-score": 0.7057237357641726, + "support": 109341.0 + }, + "cross-entropy": 0.49245265673530425, + "roc-auc": 0.6989706230822266, + "score": 0.6989706230822266 + }, + "val": { + "0": { + "precision": 0.6779895056614195, + "recall": 0.9492202603428277, + "f1-score": 0.7909998926001504, + "support": 15518.0 + }, + "1": { + "precision": 0.6891518737672584, + "recall": 0.1998169964543063, + "f1-score": 0.3098067033161908, + "support": 8743.0 + }, + "accuracy": 0.6791558468323647, + "macro avg": { + "precision": 0.683570689714339, + "recall": 0.574518628398567, + "f1-score": 0.5504032979581706, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.682012117439555, + "recall": 0.6791558468323647, + "f1-score": 0.6175910449059226, + "support": 24261.0 + }, + "cross-entropy": 0.6597667526471618, + "roc-auc": 0.6253906113125361, + "score": 0.6253906113125361 + }, + "test": { + "0": { + "precision": 0.6585443924298804, + "recall": 0.9726391679470512, + "f1-score": 0.7853513062149589, + "support": 16922.0 + }, + "1": { + "precision": 0.6833105335157319, + "recall": 0.10479387391167523, + "f1-score": 0.18171896316507502, + "support": 9533.0 + }, + "accuracy": 0.6599130599130599, + "macro avg": { + "precision": 0.6709274629728061, + "recall": 0.5387165209293632, + "f1-score": 0.48353513469001697, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6674688159026236, + "recall": 0.6599130599130599, + "f1-score": 0.5678337433234623, + "support": 26455.0 + }, + "cross-entropy": 0.7108761178650036, + "roc-auc": 0.5930931510151916, + "score": 0.5930931510151916 + } + }, + "time": "0:01:12.475731", + "diversity": { + "train": { + "mean_centered_corr": -0.013842575252056122, + "mean_pairwise_disagreement": 0.028755187603386977, + "member_std": 0.03266480937600136 + }, + "val": { + "mean_centered_corr": -0.0060155028477311134, + "mean_pairwise_disagreement": 0.08212387197825795, + "member_std": 0.06719459593296051 + }, + "test": { + "mean_centered_corr": 0.0020387060940265656, + "mean_pairwise_disagreement": 0.052944440041214236, + "member_std": 0.0697987824678421 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml new file mode 100644 index 000000000..fc9553ac2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.7 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 2 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2/report.json new file mode 100644 index 000000000..3bb7bd057 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2/report.json @@ -0,0 +1,251 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.7090908885002136, + "min_features_per_member": 65.0, + "mean_features_per_member": 78.0, + "max_features_per_member": 92.0, + "min_members_per_feature": 17.0, + "mean_members_per_feature": 22.690908432006836, + "max_members_per_feature": 28.0, + "mean_pairwise_jaccard": 0.5463052047602153, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7830380344909048, + "recall": 0.9843904061583055, + "f1-score": 0.8722447948464243, + "support": 84179.0 + }, + "1": { + "precision": 0.6262798634812287, + "recall": 0.0875129163023607, + "f1-score": 0.15356719436501848, + "support": 25162.0 + }, + "accuracy": 0.7779972745813556, + "macro avg": { + "precision": 0.7046589489860667, + "recall": 0.5359516612303331, + "f1-score": 0.5129059946057214, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7469642003486757, + "recall": 0.7779972745813556, + "f1-score": 0.7068597537062011, + "support": 109341.0 + }, + "cross-entropy": 0.4927953815266221, + "roc-auc": 0.6982500719492171, + "score": 0.6982500719492171 + }, + "val": { + "0": { + "precision": 0.6809063893016345, + "recall": 0.9449671349400696, + "f1-score": 0.7914934959788417, + "support": 15518.0 + }, + "1": { + "precision": 0.686605504587156, + "recall": 0.21399977124556788, + "f1-score": 0.32629926752703176, + "support": 8743.0 + }, + "accuracy": 0.6815465149828943, + "macro avg": { + "precision": 0.6837559469443952, + "recall": 0.5794834530928187, + "f1-score": 0.5588963817529367, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6829601943773246, + "recall": 0.6815465149828943, + "f1-score": 0.6238502356287252, + "support": 24261.0 + }, + "cross-entropy": 0.6743491854283338, + "roc-auc": 0.6220034632459894, + "score": 0.6220034632459894 + }, + "test": { + "0": { + "precision": 0.6608177720540889, + "recall": 0.9703344758302801, + "f1-score": 0.7862101987072061, + "support": 16922.0 + }, + "1": { + "precision": 0.6876166770379589, + "recall": 0.11591314381621735, + "f1-score": 0.19838420107719928, + "support": 9533.0 + }, + "accuracy": 0.6624456624456625, + "macro avg": { + "precision": 0.6742172245460238, + "recall": 0.5431238098232487, + "f1-score": 0.49229719989220266, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6704746974447988, + "recall": 0.6624456624456625, + "f1-score": 0.5743884169870453, + "support": 26455.0 + }, + "cross-entropy": 0.7345531868914768, + "roc-auc": 0.5900390978219552, + "score": 0.5900390978219552 + } + }, + "time": "0:01:07.040718", + "diversity": { + "train": { + "mean_centered_corr": -0.007606011815369129, + "mean_pairwise_disagreement": 0.026396666421797922, + "member_std": 0.03116724081337452 + }, + "val": { + "mean_centered_corr": 0.0018892597872763872, + "mean_pairwise_disagreement": 0.09870647301988722, + "member_std": 0.06490844488143921 + }, + "test": { + "mean_centered_corr": 0.01152767427265644, + "mean_pairwise_disagreement": 0.06456551295260973, + "member_std": 0.06369828432798386 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d21da633b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,255 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.6894886493682861, + "min_features_per_member": 67.0, + "mean_features_per_member": 75.84375, + "max_features_per_member": 84.0, + "min_members_per_feature": 14.0, + "mean_members_per_feature": 22.063636779785156, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.5278457608823273, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7894966160281175, + "recall": 0.9686501383955618, + "f1-score": 0.8699455883921904, + "support": 84179.0 + }, + "1": { + "precision": 0.5645214521452145, + "recall": 0.13595898577219617, + "f1-score": 0.21914034975337904, + "support": 25162.0 + }, + "accuracy": 0.7770278303655537, + "macro avg": { + "precision": 0.677009034086666, + "recall": 0.552304562083879, + "f1-score": 0.5445429690727848, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7377244073084094, + "recall": 0.7770278303655537, + "f1-score": 0.7201796139212255, + "support": 109341.0 + }, + "cross-entropy": 0.5002029863194409, + "roc-auc": 0.6821879786641952, + "score": 0.6821879786641952 + }, + "val": { + "0": { + "precision": 0.6961402811584124, + "recall": 0.9030802938523006, + "f1-score": 0.7862212123762238, + "support": 15518.0 + }, + "1": { + "precision": 0.6358353510895883, + "recall": 0.30035456936978155, + "f1-score": 0.4079857065175173, + "support": 8743.0 + }, + "accuracy": 0.6858744487036808, + "macro avg": { + "precision": 0.6659878161240004, + "recall": 0.6017174316110411, + "f1-score": 0.5971034594468705, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6744080358432264, + "recall": 0.6858744487036808, + "f1-score": 0.6499154942391862, + "support": 24261.0 + }, + "cross-entropy": 0.6154470541736884, + "roc-auc": 0.6547032702847417, + "score": 0.6547032702847417 + }, + "test": { + "0": { + "precision": 0.6733296707940667, + "recall": 0.9415553717054722, + "f1-score": 0.7851669335961562, + "support": 16922.0 + }, + "1": { + "precision": 0.6457736389684814, + "recall": 0.18913248714990036, + "f1-score": 0.2925760649087221, + "support": 9533.0 + }, + "accuracy": 0.6704214704214704, + "macro avg": { + "precision": 0.6595516548812741, + "recall": 0.5653439294276863, + "f1-score": 0.5388714992524392, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6633999164416454, + "recall": 0.6704214704214704, + "f1-score": 0.6076629172968816, + "support": 26455.0 + }, + "cross-entropy": 0.638677137478482, + "roc-auc": 0.6118564029158264, + "score": 0.6118564029158264 + } + }, + "time": "0:01:10.628940", + "diversity": { + "train": { + "mean_centered_corr": -0.015634549781680107, + "mean_pairwise_disagreement": 0.03130809179096706, + "member_std": 0.0355970673263073 + }, + "val": { + "mean_centered_corr": -0.0187542624771595, + "mean_pairwise_disagreement": 0.12205770312369114, + "member_std": 0.07598911970853806 + }, + "test": { + "mean_centered_corr": -0.018549803644418716, + "mean_pairwise_disagreement": 0.08149855811146134, + "member_std": 0.0770004391670227 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:11.162732" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..ef997fb74 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,256 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.703125, + "min_features_per_member": 67.0, + "mean_features_per_member": 77.34375, + "max_features_per_member": 88.0, + "min_members_per_feature": 14.0, + "mean_members_per_feature": 22.5, + "max_members_per_feature": 29.0, + "mean_pairwise_jaccard": 0.5422206532495487, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7910205588589639, + "recall": 0.9671652074745483, + "f1-score": 0.870269316900317, + "support": 84179.0 + }, + "1": { + "precision": 0.569269128876422, + "recall": 0.14517923853429776, + "f1-score": 0.23135628107286488, + "support": 25162.0 + }, + "accuracy": 0.7780064202815047, + "macro avg": { + "precision": 0.6801448438676929, + "recall": 0.556172223004423, + "f1-score": 0.550812798986591, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7399902090247689, + "recall": 0.7780064202815047, + "f1-score": 0.7232400249833751, + "support": 109341.0 + }, + "cross-entropy": 0.49719324451950025, + "roc-auc": 0.6874785943212433, + "score": 0.6874785943212433 + }, + "val": { + "0": { + "precision": 0.6957908163265306, + "recall": 0.9139708725351205, + "f1-score": 0.7900952593170297, + "support": 15518.0 + }, + "1": { + "precision": 0.6556615940159918, + "recall": 0.2907468832208624, + "f1-score": 0.4028526148969889, + "support": 8743.0 + }, + "accuracy": 0.6893780140966984, + "macro avg": { + "precision": 0.6757262051712611, + "recall": 0.6023588778779915, + "f1-score": 0.5964739371070094, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6813293435652659, + "recall": 0.6893780140966984, + "f1-score": 0.650543615107623, + "support": 24261.0 + }, + "cross-entropy": 0.6160387174694619, + "roc-auc": 0.6527163070467052, + "score": 0.6527163070467052 + }, + "test": { + "0": { + "precision": 0.6715682178962319, + "recall": 0.9500059094669661, + "f1-score": 0.7868820362212433, + "support": 16922.0 + }, + "1": { + "precision": 0.66388557806913, + "recall": 0.1752858491555649, + "f1-score": 0.277344398340249, + "support": 9533.0 + }, + "accuracy": 0.6708372708372708, + "macro avg": { + "precision": 0.6677268979826809, + "recall": 0.5626458793112654, + "f1-score": 0.5321132172807461, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6687997958409773, + "recall": 0.6708372708372708, + "f1-score": 0.6032712895979389, + "support": 26455.0 + }, + "cross-entropy": 0.6453489859863742, + "roc-auc": 0.6065300223672054, + "score": 0.6065300223672054 + } + }, + "time": "0:01:12.475731", + "diversity": { + "train": { + "mean_centered_corr": -0.013842575252056122, + "mean_pairwise_disagreement": 0.028755187603386977, + "member_std": 0.03266480937600136 + }, + "val": { + "mean_centered_corr": -0.0060155028477311134, + "mean_pairwise_disagreement": 0.08212387197825795, + "member_std": 0.06719459593296051 + }, + "test": { + "mean_centered_corr": 0.0020387060940265656, + "mean_pairwise_disagreement": 0.052944440041214236, + "member_std": 0.0697987824678421 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 26 + ], + "head_selection_time": "0:00:15.998123" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..898ed4405 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_mfb_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,255 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.7090908885002136, + "min_features_per_member": 65.0, + "mean_features_per_member": 78.0, + "max_features_per_member": 92.0, + "min_members_per_feature": 17.0, + "mean_members_per_feature": 22.690908432006836, + "max_members_per_feature": 28.0, + "mean_pairwise_jaccard": 0.5463052047602153, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7825240884186662, + "recall": 0.9840696610793666, + "f1-score": 0.8717999989475844, + "support": 84179.0 + }, + "1": { + "precision": 0.6147658718758977, + "recall": 0.08504888323662665, + "f1-score": 0.14942568864993191, + "support": 25162.0 + }, + "accuracy": 0.7771833072680879, + "macro avg": { + "precision": 0.6986449801472819, + "recall": 0.5345592721579966, + "f1-score": 0.5106128437987582, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439188786195136, + "recall": 0.7771833072680879, + "f1-score": 0.7055642557615013, + "support": 109341.0 + }, + "cross-entropy": 0.49927567921826405, + "roc-auc": 0.6841046410049181, + "score": 0.6841046410049181 + }, + "val": { + "0": { + "precision": 0.6951219512195121, + "recall": 0.9256347467457147, + "f1-score": 0.7939859598695483, + "support": 15518.0 + }, + "1": { + "precision": 0.6791770920211287, + "recall": 0.27942353883106485, + "f1-score": 0.3959481361426256, + "support": 8743.0 + }, + "accuracy": 0.6927579242405507, + "macro avg": { + "precision": 0.6871495216203205, + "recall": 0.6025291427883898, + "f1-score": 0.5949670480060869, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.689375860622609, + "recall": 0.6927579242405507, + "f1-score": 0.6505440286694953, + "support": 24261.0 + }, + "cross-entropy": 0.6149192758236253, + "roc-auc": 0.6546592087434608, + "score": 0.6546592087434608 + }, + "test": { + "0": { + "precision": 0.6737775546910947, + "recall": 0.9518969388961116, + "f1-score": 0.7890470009062186, + "support": 16922.0 + }, + "1": { + "precision": 0.6805337519623234, + "recall": 0.18189447183467952, + "f1-score": 0.2870623292773777, + "support": 9533.0 + }, + "accuracy": 0.6744282744282745, + "macro avg": { + "precision": 0.677155653326709, + "recall": 0.5668957053653956, + "f1-score": 0.5380546650917981, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6762121352462495, + "recall": 0.6744282744282745, + "f1-score": 0.6081579487558598, + "support": 26455.0 + }, + "cross-entropy": 0.6464507795618373, + "roc-auc": 0.6088559366177836, + "score": 0.6088559366177836 + } + }, + "time": "0:01:07.040718", + "diversity": { + "train": { + "mean_centered_corr": -0.007606011815369129, + "mean_pairwise_disagreement": 0.026396666421797922, + "member_std": 0.03116724081337452 + }, + "val": { + "mean_centered_corr": 0.0018892597872763872, + "mean_pairwise_disagreement": 0.09870647301988722, + "member_std": 0.06490844488143921 + }, + "test": { + "mean_centered_corr": 0.01152767427265644, + "mean_pairwise_disagreement": 0.06456551295260973, + "member_std": 0.06369828432798386 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:11.198245" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..16776bce5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.779700580278046, + "recall": 0.9880492759476829, + "f1-score": 0.8715968394357931, + "support": 84179.0 + }, + "1": { + "precision": 0.6229385307346327, + "recall": 0.06605198314919322, + "f1-score": 0.11943945382680561, + "support": 25162.0 + }, + "accuracy": 0.7758754721467702, + "macro avg": { + "precision": 0.7013195555063394, + "recall": 0.5270506295484381, + "f1-score": 0.4955181466312994, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436258535917035, + "recall": 0.7758754721467702, + "f1-score": 0.6985072926354772, + "support": 109341.0 + }, + "cross-entropy": 0.5001462491291722, + "roc-auc": 0.6828783854988578, + "score": 0.6828783854988578 + }, + "val": { + "0": { + "precision": 0.6734214920792526, + "recall": 0.9615285474932337, + "f1-score": 0.7920902455209025, + "support": 15518.0 + }, + "1": { + "precision": 0.716254752851711, + "recall": 0.17236646460025162, + "f1-score": 0.2778648474232507, + "support": 8743.0 + }, + "accuracy": 0.677136144429331, + "macro avg": { + "precision": 0.6948381224654818, + "recall": 0.5669475060467426, + "f1-score": 0.5349775464720765, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.688857426250705, + "recall": 0.677136144429331, + "f1-score": 0.6067774943742981, + "support": 24261.0 + }, + "cross-entropy": 0.6457686779603163, + "roc-auc": 0.6396541901648656, + "score": 0.6396541901648656 + }, + "test": { + "0": { + "precision": 0.6569502280388657, + "recall": 0.9789032029310956, + "f1-score": 0.7862448679307972, + "support": 16922.0 + }, + "1": { + "precision": 0.7120967741935483, + "recall": 0.09262561628028952, + "f1-score": 0.16392833936693585, + "support": 9533.0 + }, + "accuracy": 0.6595350595350595, + "macro avg": { + "precision": 0.684523501116207, + "recall": 0.5357644096056925, + "f1-score": 0.4750866036488665, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.67682216243662, + "recall": 0.6595350595350595, + "f1-score": 0.5619945006354167, + "support": 26455.0 + }, + "cross-entropy": 0.6822343746621596, + "roc-auc": 0.6015418321886687, + "score": 0.6015418321886687 + } + }, + "time": "0:01:18.218817", + "diversity": { + "train": { + "mean_centered_corr": -0.016645358875393867, + "mean_pairwise_disagreement": 0.024673918911862297, + "member_std": 0.027964944019913673 + }, + "val": { + "mean_centered_corr": 0.002000282984226942, + "mean_pairwise_disagreement": 0.06748003233651247, + "member_std": 0.05873498693108559 + }, + "test": { + "mean_centered_corr": 0.008286470547318459, + "mean_pairwise_disagreement": 0.04453758969888002, + "member_std": 0.06023459881544113 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.887253" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..c19a0f6c2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7922289232927696, + "recall": 0.9659416243956331, + "f1-score": 0.8705035971223022, + "support": 84179.0 + }, + "1": { + "precision": 0.5723448687350835, + "recall": 0.15249185279389554, + "f1-score": 0.24082093767652044, + "support": 25162.0 + }, + "accuracy": 0.7787472219935797, + "macro avg": { + "precision": 0.6822868960139266, + "recall": 0.5592167385947644, + "f1-score": 0.5556622673994113, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7416283015609353, + "recall": 0.7787472219935797, + "f1-score": 0.72559843732886, + "support": 109341.0 + }, + "cross-entropy": 0.49316481374570875, + "roc-auc": 0.695882539918458, + "score": 0.695882539918458 + }, + "val": { + "0": { + "precision": 0.6824891054777189, + "recall": 0.9385874468359324, + "f1-score": 0.7903090154372067, + "support": 15518.0 + }, + "1": { + "precision": 0.6736301369863014, + "recall": 0.22497998398718974, + "f1-score": 0.3373060104604304, + "support": 8743.0 + }, + "accuracy": 0.6814228597337291, + "macro avg": { + "precision": 0.6780596212320101, + "recall": 0.581783715411561, + "f1-score": 0.5638075129488185, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.679296575840834, + "recall": 0.6814228597337291, + "f1-score": 0.6270591381645486, + "support": 24261.0 + }, + "cross-entropy": 0.6646895058244983, + "roc-auc": 0.6311484553024557, + "score": 0.6311484553024557 + }, + "test": { + "0": { + "precision": 0.6616923449221586, + "recall": 0.969507150455029, + "f1-score": 0.7865567168472528, + "support": 16922.0 + }, + "1": { + "precision": 0.6893437688139675, + "recall": 0.12010909472359173, + "f1-score": 0.20457387886367698, + "support": 9533.0 + }, + "accuracy": 0.6634284634284634, + "macro avg": { + "precision": 0.675518056868063, + "recall": 0.5448081225893103, + "f1-score": 0.4955652978554649, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6716564735919985, + "recall": 0.6634284634284634, + "f1-score": 0.5768405046190378, + "support": 26455.0 + }, + "cross-entropy": 0.7219351511377476, + "roc-auc": 0.5960986415689524, + "score": 0.5960986415689524 + } + }, + "time": "0:01:28.118874", + "diversity": { + "train": { + "mean_centered_corr": -0.010014320723712444, + "mean_pairwise_disagreement": 0.02492483562079095, + "member_std": 0.024375617504119873 + }, + "val": { + "mean_centered_corr": 0.02006583847105503, + "mean_pairwise_disagreement": 0.06821390297184782, + "member_std": 0.04597385972738266 + }, + "test": { + "mean_centered_corr": 0.03903863951563835, + "mean_pairwise_disagreement": 0.03914178672243188, + "member_std": 0.04212528467178345 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 15 + ], + "head_selection_time": "0:00:04.875768" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..d4aba5fe6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7856289713821605, + "recall": 0.9767281626058756, + "f1-score": 0.8708177552771217, + "support": 84179.0 + }, + "1": { + "precision": 0.5819462227912933, + "recall": 0.10837771242349575, + "f1-score": 0.18272581077459127, + "support": 25162.0 + }, + "accuracy": 0.7768997905634666, + "macro avg": { + "precision": 0.6837875970867269, + "recall": 0.5425529375146857, + "f1-score": 0.5267717830258565, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7387566607206209, + "recall": 0.7768997905634666, + "f1-score": 0.7124712109106656, + "support": 109341.0 + }, + "cross-entropy": 0.49964959969501876, + "roc-auc": 0.6794902287315214, + "score": 0.6794902287315214 + }, + "val": { + "0": { + "precision": 0.6822470605312817, + "recall": 0.9086222451346823, + "f1-score": 0.7793284510156142, + "support": 15518.0 + }, + "1": { + "precision": 0.6054535336672231, + "recall": 0.24888482214342902, + "f1-score": 0.352759990273162, + "support": 8743.0 + }, + "accuracy": 0.6708709451382878, + "macro avg": { + "precision": 0.6438502970992523, + "recall": 0.5787535336390557, + "f1-score": 0.5660442206443881, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6545727764798219, + "recall": 0.6708709451382878, + "f1-score": 0.6256048595613766, + "support": 24261.0 + }, + "cross-entropy": 0.6436639930347295, + "roc-auc": 0.633681175787757, + "score": 0.633681175787757 + }, + "test": { + "0": { + "precision": 0.6632580953568608, + "recall": 0.9429145491076705, + "f1-score": 0.7787403304131384, + "support": 16922.0 + }, + "1": { + "precision": 0.5971643035863219, + "recall": 0.15021504248400294, + "f1-score": 0.24004693655183976, + "support": 9533.0 + }, + "accuracy": 0.6572670572670573, + "macro avg": { + "precision": 0.6302111994715913, + "recall": 0.5465647957958367, + "f1-score": 0.5093936334824891, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6394413455194559, + "recall": 0.6572670572670573, + "f1-score": 0.5846233724210854, + "support": 26455.0 + }, + "cross-entropy": 0.6816832612823018, + "roc-auc": 0.597700306723218, + "score": 0.597700306723218 + } + }, + "time": "0:01:22.843981", + "diversity": { + "train": { + "mean_centered_corr": -0.01568707637488842, + "mean_pairwise_disagreement": 0.023222997836599382, + "member_std": 0.024020660668611526 + }, + "val": { + "mean_centered_corr": -0.00023190633510239422, + "mean_pairwise_disagreement": 0.07023343917158961, + "member_std": 0.050259560346603394 + }, + "test": { + "mean_centered_corr": 0.006958201061934233, + "mean_pairwise_disagreement": 0.04134691594369014, + "member_std": 0.04727793484926224 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 20 + ], + "head_selection_time": "0:00:04.821733" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml new file mode 100644 index 000000000..2990735ec --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/0/report.json new file mode 100644 index 000000000..1b45b9b2d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7813988964010622, + "recall": 0.9857921809477423, + "f1-score": 0.8717754770796893, + "support": 84179.0 + }, + "1": { + "precision": 0.6194718421889914, + "recall": 0.0773785867578094, + "f1-score": 0.1375728669846317, + "support": 25162.0 + }, + "accuracy": 0.7767443136609323, + "macro avg": { + "precision": 0.7004353692950268, + "recall": 0.5315853838527759, + "f1-score": 0.5046741720321605, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.744135577626914, + "recall": 0.7767443136609323, + "f1-score": 0.7028177569636135, + "support": 109341.0 + }, + "cross-entropy": 0.4956556906392919, + "roc-auc": 0.6916462356019382, + "score": 0.6916462356019382 + }, + "val": { + "0": { + "precision": 0.6749512449544197, + "recall": 0.9590153370279675, + "f1-score": 0.7922911065562861, + "support": 15518.0 + }, + "1": { + "precision": 0.7124773960216998, + "recall": 0.18025849250829235, + "f1-score": 0.2877225011410315, + "support": 8743.0 + }, + "accuracy": 0.6783726969209843, + "macro avg": { + "precision": 0.6937143204880598, + "recall": 0.5696369147681299, + "f1-score": 0.5400068038486587, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6884746421260627, + "recall": 0.6783726969209843, + "f1-score": 0.6104583990361686, + "support": 24261.0 + }, + "cross-entropy": 0.6750056416383268, + "roc-auc": 0.6227001707049361, + "score": 0.6227001707049361 + }, + "test": { + "0": { + "precision": 0.6569412510901451, + "recall": 0.9793168656187212, + "f1-score": 0.7863718325899213, + "support": 16922.0 + }, + "1": { + "precision": 0.7152156224572823, + "recall": 0.09220602118955208, + "f1-score": 0.163352536703215, + "support": 9533.0 + }, + "accuracy": 0.6596484596484596, + "macro avg": { + "precision": 0.6860784367737137, + "recall": 0.5357614434041367, + "f1-score": 0.47486218464656815, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6779402902979667, + "recall": 0.6596484596484596, + "f1-score": 0.561868224663708, + "support": 26455.0 + }, + "cross-entropy": 0.7361612281771354, + "roc-auc": 0.5911765632808944, + "score": 0.5911765632808944 + } + }, + "time": "0:01:18.218817", + "diversity": { + "train": { + "mean_centered_corr": -0.016645358875393867, + "mean_pairwise_disagreement": 0.024673918911862297, + "member_std": 0.027964944019913673 + }, + "val": { + "mean_centered_corr": 0.002000282984226942, + "mean_pairwise_disagreement": 0.06748003233651247, + "member_std": 0.05873498693108559 + }, + "test": { + "mean_centered_corr": 0.008286470547318459, + "mean_pairwise_disagreement": 0.04453758969888002, + "member_std": 0.06023459881544113 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml new file mode 100644 index 000000000..1cf01543f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/1/report.json new file mode 100644 index 000000000..434ecb008 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/1/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7881281081392446, + "recall": 0.9752194727901258, + "f1-score": 0.8717485836859739, + "support": 84179.0 + }, + "1": { + "precision": 0.5972195404518247, + "recall": 0.12292345600508704, + "f1-score": 0.20388253518341518, + "support": 25162.0 + }, + "accuracy": 0.7790856128990955, + "macro avg": { + "precision": 0.6926738242955346, + "recall": 0.5490714643976065, + "f1-score": 0.5378155594346945, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441954444526964, + "recall": 0.7790856128990955, + "f1-score": 0.7180565055778407, + "support": 109341.0 + }, + "cross-entropy": 0.4903386782360664, + "roc-auc": 0.7032463632265398, + "score": 0.7032463632265398 + }, + "val": { + "0": { + "precision": 0.6736268724466636, + "recall": 0.9563088026807578, + "f1-score": 0.790454884414616, + "support": 15518.0 + }, + "1": { + "precision": 0.6961004034065441, + "recall": 0.17762781653894544, + "f1-score": 0.2830326225624203, + "support": 8743.0 + }, + "accuracy": 0.6756934998557356, + "macro avg": { + "precision": 0.6848636379266039, + "recall": 0.5669683096098517, + "f1-score": 0.5367437534885181, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6817257175553663, + "recall": 0.6756934998557356, + "f1-score": 0.6075937972634785, + "support": 24261.0 + }, + "cross-entropy": 0.715360677146401, + "roc-auc": 0.6203498803314189, + "score": 0.6203498803314189 + }, + "test": { + "0": { + "precision": 0.655632548010295, + "recall": 0.97848954024347, + "f1-score": 0.785167271261588, + "support": 16922.0 + }, + "1": { + "precision": 0.6966666666666667, + "recall": 0.08769537396412462, + "f1-score": 0.15578123544209446, + "support": 9533.0 + }, + "accuracy": 0.6574938574938575, + "macro avg": { + "precision": 0.6761496073384808, + "recall": 0.5330924571037973, + "f1-score": 0.47047425335184123, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6704191007659627, + "recall": 0.6574938574938575, + "f1-score": 0.5583694228598782, + "support": 26455.0 + }, + "cross-entropy": 0.8001816812510966, + "roc-auc": 0.5897023859034299, + "score": 0.5897023859034299 + } + }, + "time": "0:01:28.118874", + "diversity": { + "train": { + "mean_centered_corr": -0.010014320723712444, + "mean_pairwise_disagreement": 0.02492483562079095, + "member_std": 0.024375617504119873 + }, + "val": { + "mean_centered_corr": 0.02006583847105503, + "mean_pairwise_disagreement": 0.06821390297184782, + "member_std": 0.04597385972738266 + }, + "test": { + "mean_centered_corr": 0.03903863951563835, + "mean_pairwise_disagreement": 0.03914178672243188, + "member_std": 0.04212528467178345 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml new file mode 100644 index 000000000..d7e69509b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/2/report.json new file mode 100644 index 000000000..acd01dc37 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-evaluation/2/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.786029867837206, + "recall": 0.9785338386058281, + "f1-score": 0.8717813032480659, + "support": 84179.0 + }, + "1": { + "precision": 0.6025076990761109, + "recall": 0.10885462204912169, + "f1-score": 0.1843947758179615, + "support": 25162.0 + }, + "accuracy": 0.7783996853879148, + "macro avg": { + "precision": 0.6942687834566584, + "recall": 0.5436942303274749, + "f1-score": 0.5280880395330136, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7437969926086396, + "recall": 0.7783996853879148, + "f1-score": 0.713597092355571, + "support": 109341.0 + }, + "cross-entropy": 0.491904955768715, + "roc-auc": 0.699991829704937, + "score": 0.699991829704937 + }, + "val": { + "0": { + "precision": 0.6761661574939584, + "recall": 0.9555999484469648, + "f1-score": 0.7919570616037811, + "support": 15518.0 + }, + "1": { + "precision": 0.7042918454935623, + "recall": 0.18769301155209883, + "f1-score": 0.29639664047683556, + "support": 8743.0 + }, + "accuracy": 0.6788673179176457, + "macro avg": { + "precision": 0.6902290014937603, + "recall": 0.5716464799995318, + "f1-score": 0.5441768510403083, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6863018852125411, + "recall": 0.6788673179176457, + "f1-score": 0.6133706570073966, + "support": 24261.0 + }, + "cross-entropy": 0.7034186632975251, + "roc-auc": 0.6211526583224122, + "score": 0.6211526583224122 + }, + "test": { + "0": { + "precision": 0.6576497732516509, + "recall": 0.9769530788322893, + "f1-score": 0.7861150737042321, + "support": 16922.0 + }, + "1": { + "precision": 0.7038724373576309, + "recall": 0.09724116227840135, + "f1-score": 0.17087557603686637, + "support": 9533.0 + }, + "accuracy": 0.65995085995086, + "macro avg": { + "precision": 0.680761105304641, + "recall": 0.5370971205553453, + "f1-score": 0.47849532487054924, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6743060067395475, + "recall": 0.65995085995086, + "f1-score": 0.5644148986423158, + "support": 26455.0 + }, + "cross-entropy": 0.781592470979893, + "roc-auc": 0.5892307660549951, + "score": 0.5892307660549951 + } + }, + "time": "0:01:22.843981", + "diversity": { + "train": { + "mean_centered_corr": -0.01568707637488842, + "mean_pairwise_disagreement": 0.023222997836599382, + "member_std": 0.024020660668611526 + }, + "val": { + "mean_centered_corr": -0.00023190633510239422, + "mean_pairwise_disagreement": 0.07023343917158961, + "member_std": 0.050259560346603394 + }, + "test": { + "mean_centered_corr": 0.006958201061934233, + "mean_pairwise_disagreement": 0.04134691594369014, + "member_std": 0.04727793484926224 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ec841c0fe --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.779700580278046, + "recall": 0.9880492759476829, + "f1-score": 0.8715968394357931, + "support": 84179.0 + }, + "1": { + "precision": 0.6229385307346327, + "recall": 0.06605198314919322, + "f1-score": 0.11943945382680561, + "support": 25162.0 + }, + "accuracy": 0.7758754721467702, + "macro avg": { + "precision": 0.7013195555063394, + "recall": 0.5270506295484381, + "f1-score": 0.4955181466312994, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436258535917035, + "recall": 0.7758754721467702, + "f1-score": 0.6985072926354772, + "support": 109341.0 + }, + "cross-entropy": 0.5001462491291722, + "roc-auc": 0.6828783854988578, + "score": 0.6828783854988578 + }, + "val": { + "0": { + "precision": 0.6734214920792526, + "recall": 0.9615285474932337, + "f1-score": 0.7920902455209025, + "support": 15518.0 + }, + "1": { + "precision": 0.716254752851711, + "recall": 0.17236646460025162, + "f1-score": 0.2778648474232507, + "support": 8743.0 + }, + "accuracy": 0.677136144429331, + "macro avg": { + "precision": 0.6948381224654818, + "recall": 0.5669475060467426, + "f1-score": 0.5349775464720765, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.688857426250705, + "recall": 0.677136144429331, + "f1-score": 0.6067774943742981, + "support": 24261.0 + }, + "cross-entropy": 0.6457686779603163, + "roc-auc": 0.6396541901648656, + "score": 0.6396541901648656 + }, + "test": { + "0": { + "precision": 0.6569502280388657, + "recall": 0.9789032029310956, + "f1-score": 0.7862448679307972, + "support": 16922.0 + }, + "1": { + "precision": 0.7120967741935483, + "recall": 0.09262561628028952, + "f1-score": 0.16392833936693585, + "support": 9533.0 + }, + "accuracy": 0.6595350595350595, + "macro avg": { + "precision": 0.684523501116207, + "recall": 0.5357644096056925, + "f1-score": 0.4750866036488665, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.67682216243662, + "recall": 0.6595350595350595, + "f1-score": 0.5619945006354167, + "support": 26455.0 + }, + "cross-entropy": 0.6822343746621596, + "roc-auc": 0.6015418321886687, + "score": 0.6015418321886687 + } + }, + "time": "0:01:18.218817", + "diversity": { + "train": { + "mean_centered_corr": -0.016645358875393867, + "mean_pairwise_disagreement": 0.024673918911862297, + "member_std": 0.027964944019913673 + }, + "val": { + "mean_centered_corr": 0.002000282984226942, + "mean_pairwise_disagreement": 0.06748003233651247, + "member_std": 0.05873498693108559 + }, + "test": { + "mean_centered_corr": 0.008286470547318459, + "mean_pairwise_disagreement": 0.04453758969888002, + "member_std": 0.06023459881544113 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.224192" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..dd3929da1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,232 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7934275978222552, + "recall": 0.9625678613430904, + "f1-score": 0.869851800561451, + "support": 84179.0 + }, + "1": { + "precision": 0.5633919911320493, + "recall": 0.16159287814959064, + "f1-score": 0.25115043701164336, + "support": 25162.0 + }, + "accuracy": 0.7782442084853806, + "macro avg": { + "precision": 0.6784097944771523, + "recall": 0.5620803697463405, + "f1-score": 0.5605011187865472, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7404908592197277, + "recall": 0.7782442084853806, + "f1-score": 0.727473701681431, + "support": 109341.0 + }, + "cross-entropy": 0.4936760184787449, + "roc-auc": 0.6962309860821628, + "score": 0.6962309860821628 + }, + "val": { + "0": { + "precision": 0.6906453952139231, + "recall": 0.9206083258151824, + "f1-score": 0.7892163632848106, + "support": 15518.0 + }, + "1": { + "precision": 0.6554809843400448, + "recall": 0.2681001944412673, + "f1-score": 0.3805503693481614, + "support": 8743.0 + }, + "accuracy": 0.6854622645397964, + "macro avg": { + "precision": 0.6730631897769839, + "recall": 0.5943542601282248, + "f1-score": 0.584883366316486, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6779731045305085, + "recall": 0.6854622645397964, + "f1-score": 0.641944330599096, + "support": 24261.0 + }, + "cross-entropy": 0.6564983741679696, + "roc-auc": 0.6323603724914644, + "score": 0.6323603724914644 + }, + "test": { + "0": { + "precision": 0.6652895942408377, + "recall": 0.9611748020328567, + "f1-score": 0.7863185883490452, + "support": 16922.0 + }, + "1": { + "precision": 0.672645739910314, + "recall": 0.14161334312388546, + "f1-score": 0.2339688041594454, + "support": 9533.0 + }, + "accuracy": 0.6658476658476659, + "macro avg": { + "precision": 0.6689676670755758, + "recall": 0.551394072578371, + "f1-score": 0.5101436962542453, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6679403648576253, + "recall": 0.6658476658476659, + "f1-score": 0.5872805806877542, + "support": 26455.0 + }, + "cross-entropy": 0.7071797390975189, + "roc-auc": 0.5968416363152236, + "score": 0.5968416363152236 + } + }, + "time": "0:01:28.118874", + "diversity": { + "train": { + "mean_centered_corr": -0.010014320723712444, + "mean_pairwise_disagreement": 0.02492483562079095, + "member_std": 0.024375617504119873 + }, + "val": { + "mean_centered_corr": 0.02006583847105503, + "mean_pairwise_disagreement": 0.06821390297184782, + "member_std": 0.04597385972738266 + }, + "test": { + "mean_centered_corr": 0.03903863951563835, + "mean_pairwise_disagreement": 0.03914178672243188, + "member_std": 0.04212528467178345 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 15, + 10, + 9, + 20, + 9 + ], + "head_selection_time": "0:00:30.228414" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..174679548 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/ecom-offers/best_rla_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7871367345177713, + "recall": 0.9752551111322302, + "f1-score": 0.8711560092531675, + "support": 84179.0 + }, + "1": { + "precision": 0.5870340999206979, + "recall": 0.11767745012320166, + "f1-score": 0.19605376415281733, + "support": 25162.0 + }, + "accuracy": 0.7779058175798649, + "macro avg": { + "precision": 0.6870854172192347, + "recall": 0.546466280627716, + "f1-score": 0.5336048867029924, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7410882943925524, + "recall": 0.7779058175798649, + "f1-score": 0.7157987078637984, + "support": 109341.0 + }, + "cross-entropy": 0.4982793043881891, + "roc-auc": 0.6827272079405879, + "score": 0.6827272079405879 + }, + "val": { + "0": { + "precision": 0.689112461400538, + "recall": 0.89160974352365, + "f1-score": 0.7773907180582088, + "support": 15518.0 + }, + "1": { + "precision": 0.5978962467128854, + "recall": 0.2860574173624614, + "f1-score": 0.3869719944298313, + "support": 8743.0 + }, + "accuracy": 0.6733852685379828, + "macro avg": { + "precision": 0.6435043540567117, + "recall": 0.5888335804430557, + "f1-score": 0.5821813562440201, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.656240635630201, + "recall": 0.6733852685379828, + "f1-score": 0.6366945018806851, + "support": 24261.0 + }, + "cross-entropy": 0.6339658157770617, + "roc-auc": 0.6351641473729864, + "score": 0.6351641473729864 + }, + "test": { + "0": { + "precision": 0.6670301098716765, + "recall": 0.9399598156246307, + "f1-score": 0.7803178963893249, + "support": 16922.0 + }, + "1": { + "precision": 0.6105787658106554, + "recall": 0.16710374488618482, + "f1-score": 0.2623949925877121, + "support": 9533.0 + }, + "accuracy": 0.6614628614628615, + "macro avg": { + "precision": 0.6388044378411659, + "recall": 0.5535317802554077, + "f1-score": 0.5213564444885185, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6466879944706667, + "recall": 0.6614628614628615, + "f1-score": 0.5936855379716052, + "support": 26455.0 + }, + "cross-entropy": 0.6638257558023992, + "roc-auc": 0.5998596611627065, + "score": 0.5998596611627065 + } + }, + "time": "0:01:22.843981", + "diversity": { + "train": { + "mean_centered_corr": -0.01568707637488842, + "mean_pairwise_disagreement": 0.023222997836599382, + "member_std": 0.024020660668611526 + }, + "val": { + "mean_centered_corr": -0.00023190633510239422, + "mean_pairwise_disagreement": 0.07023343917158961, + "member_std": 0.050259560346603394 + }, + "test": { + "mean_centered_corr": 0.006958201061934233, + "mean_pairwise_disagreement": 0.04134691594369014, + "member_std": 0.04727793484926224 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 20, + 31 + ], + "head_selection_time": "0:00:15.838549" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..67ae69ad9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,262 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9509438009438009, + "recall": 0.9745069269279171, + "f1-score": 0.9625811844722908, + "support": 181971.0 + }, + "1": { + "precision": 0.8774048625792812, + "recall": 0.7839854542019883, + "f1-score": 0.8280686877252491, + "support": 42349.0 + }, + "accuracy": 0.9385386947218259, + "macro avg": { + "precision": 0.9141743317615411, + "recall": 0.8792461905649527, + "f1-score": 0.89532493609877, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9370605069851746, + "recall": 0.9385386947218259, + "f1-score": 0.9371867937592895, + "support": 224320.0 + }, + "cross-entropy": 0.14097183098455576, + "roc-auc": 0.9772540623630491, + "score": 0.9772540623630491 + }, + "val": { + "0": { + "precision": 0.9410365039753932, + "recall": 0.9685801326085658, + "f1-score": 0.9546096785588132, + "support": 16741.0 + }, + "1": { + "precision": 0.8190574475404196, + "recall": 0.7009125699146306, + "f1-score": 0.7553934010152284, + "support": 3397.0 + }, + "accuracy": 0.923428344423478, + "macro avg": { + "precision": 0.8800469757579064, + "recall": 0.8347463512615982, + "f1-score": 0.8550015397870208, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9204603367934683, + "recall": 0.923428344423478, + "f1-score": 0.9210046683882125, + "support": 20138.0 + }, + "cross-entropy": 0.181709302619563, + "roc-auc": 0.9554763470552776, + "score": 0.9554763470552776 + }, + "test": { + "0": { + "precision": 0.9398939744642724, + "recall": 0.9574807940975127, + "f1-score": 0.9486058779201205, + "support": 13147.0 + }, + "1": { + "precision": 0.8073742246726395, + "recall": 0.7442820838627701, + "f1-score": 0.7745454545454545, + "support": 3148.0 + }, + "accuracy": 0.9162933415158024, + "macro avg": { + "precision": 0.873634099568456, + "recall": 0.8508814389801413, + "f1-score": 0.8615756662327876, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9142927365174138, + "recall": 0.9162933415158024, + "f1-score": 0.914979476399197, + "support": 16295.0 + }, + "cross-entropy": 0.18955508005758365, + "roc-auc": 0.959416932798502, + "score": 0.959416932798502 + } + }, + "time": "0:02:48.403690", + "diversity": { + "train": { + "mean_centered_corr": 0.05291927233338356, + "mean_pairwise_disagreement": 0.03400679940235148, + "member_std": 0.03158120810985565 + }, + "val": { + "mean_centered_corr": 0.06288693100214005, + "mean_pairwise_disagreement": 0.03244768356405319, + "member_std": 0.0349886491894722 + }, + "test": { + "mean_centered_corr": 0.07999585568904877, + "mean_pairwise_disagreement": 0.041502439893495925, + "member_std": 0.0369756743311882 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.916152" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/1/report.json new file mode 100644 index 000000000..1bddbac12 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/1/report.json @@ -0,0 +1,262 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9421781614647566, + "recall": 0.9823927988525645, + "f1-score": 0.9618653301372848, + "support": 181971.0 + }, + "1": { + "precision": 0.9073506448441385, + "recall": 0.7409383928782262, + "f1-score": 0.8157439783702278, + "support": 42349.0 + }, + "accuracy": 0.9368090228245364, + "macro avg": { + "precision": 0.9247644031544475, + "recall": 0.8616655958653954, + "f1-score": 0.8888046542537563, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9356031324822024, + "recall": 0.9368090228245364, + "f1-score": 0.934279322977945, + "support": 224320.0 + }, + "cross-entropy": 0.1448925441874488, + "roc-auc": 0.975466291176613, + "score": 0.975466291176613 + }, + "val": { + "0": { + "precision": 0.9327206716587247, + "recall": 0.9821396571292037, + "f1-score": 0.9567924583199977, + "support": 16741.0 + }, + "1": { + "precision": 0.8808764940239043, + "recall": 0.6508684133058581, + "f1-score": 0.7486033519553073, + "support": 3397.0 + }, + "accuracy": 0.9262588141821432, + "macro avg": { + "precision": 0.9067985828413145, + "recall": 0.8165040352175309, + "f1-score": 0.8526979051376524, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9239752812810563, + "recall": 0.9262588141821432, + "f1-score": 0.9216738569533847, + "support": 20138.0 + }, + "cross-entropy": 0.17975617136163471, + "roc-auc": 0.9563250405399748, + "score": 0.9563250405399748 + }, + "test": { + "0": { + "precision": 0.9304106791159092, + "recall": 0.9701833117821557, + "f1-score": 0.9498808459934466, + "support": 13147.0 + }, + "1": { + "precision": 0.848414539829853, + "recall": 0.6969504447268107, + "f1-score": 0.7652598535054064, + "support": 3148.0 + }, + "accuracy": 0.9173979748389076, + "macro avg": { + "precision": 0.8894126094728811, + "recall": 0.8335668782544832, + "f1-score": 0.8575703497494265, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9145700012102631, + "recall": 0.9173979748389076, + "f1-score": 0.9142142682485954, + "support": 16295.0 + }, + "cross-entropy": 0.1871619815011916, + "roc-auc": 0.9592494275221765, + "score": 0.9592494275221765 + } + }, + "time": "0:02:44.189369", + "diversity": { + "train": { + "mean_centered_corr": 0.05150473117828369, + "mean_pairwise_disagreement": 0.03169338301274677, + "member_std": 0.03007516823709011 + }, + "val": { + "mean_centered_corr": 0.057541172951459885, + "mean_pairwise_disagreement": 0.02847489419777727, + "member_std": 0.031669627875089645 + }, + "test": { + "mean_centered_corr": 0.06821995228528976, + "mean_pairwise_disagreement": 0.035880044343703295, + "member_std": 0.033187758177518845 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:01.162128" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/2/report.json new file mode 100644 index 000000000..28b72c8fb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-best-head-evaluation/2/report.json @@ -0,0 +1,262 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9441356426540986, + "recall": 0.981964159124256, + "f1-score": 0.9626784256787516, + "support": 181971.0 + }, + "1": { + "precision": 0.9063837069998288, + "recall": 0.7503364896455642, + "f1-score": 0.8210110196752232, + "support": 42349.0 + }, + "accuracy": 0.9382355563480742, + "macro avg": { + "precision": 0.9252596748269637, + "recall": 0.8661503243849101, + "f1-score": 0.8918447226769874, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9370085174623071, + "recall": 0.9382355563480742, + "f1-score": 0.9359332715380446, + "support": 224320.0 + }, + "cross-entropy": 0.14035186091185498, + "roc-auc": 0.977574758838111, + "score": 0.977574758838111 + }, + "val": { + "0": { + "precision": 0.9334280841387151, + "recall": 0.980765784600681, + "f1-score": 0.9565116075849815, + "support": 16741.0 + }, + "1": { + "precision": 0.8736263736263736, + "recall": 0.6552840741831027, + "f1-score": 0.7488645920941968, + "support": 3397.0 + }, + "accuracy": 0.9258615552686463, + "macro avg": { + "precision": 0.9035272288825444, + "recall": 0.8180249293918919, + "f1-score": 0.8526880998395892, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9233403688437293, + "recall": 0.9258615552686463, + "f1-score": 0.92148444939538, + "support": 20138.0 + }, + "cross-entropy": 0.1855825324099513, + "roc-auc": 0.9561054927874199, + "score": 0.9561054927874199 + }, + "test": { + "0": { + "precision": 0.9291811342592593, + "recall": 0.9770289799954363, + "f1-score": 0.9525045419153906, + "support": 13147.0 + }, + "1": { + "precision": 0.877782274382841, + "recall": 0.6890088945362135, + "f1-score": 0.7720234917245061, + "support": 3148.0 + }, + "accuracy": 0.9213869285056766, + "macro avg": { + "precision": 0.9034817043210501, + "recall": 0.8330189372658249, + "f1-score": 0.8622640168199484, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9192514864598752, + "recall": 0.9213869285056766, + "f1-score": 0.9176377517342981, + "support": 16295.0 + }, + "cross-entropy": 0.18959449862593228, + "roc-auc": 0.9608390423255206, + "score": 0.9608390423255206 + } + }, + "time": "0:02:48.224028", + "diversity": { + "train": { + "mean_centered_corr": 0.05968523025512695, + "mean_pairwise_disagreement": 0.033705988852330775, + "member_std": 0.031936075538396835 + }, + "val": { + "mean_centered_corr": 0.06932749599218369, + "mean_pairwise_disagreement": 0.031030346255994927, + "member_std": 0.03422032669186592 + }, + "test": { + "mean_centered_corr": 0.08440784364938736, + "mean_pairwise_disagreement": 0.03857926931871047, + "member_std": 0.03628651052713394 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1 + ], + "head_selection_time": "0:00:01.010209" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..d85b237a3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml @@ -0,0 +1,34 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..cd297e4bc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0/report.json @@ -0,0 +1,258 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9482836172363248, + "recall": 0.984772298882789, + "f1-score": 0.966183574879227, + "support": 181971.0 + }, + "1": { + "precision": 0.9216057939853453, + "recall": 0.769227136414083, + "f1-score": 0.8385502471169687, + "support": 42349.0 + }, + "accuracy": 0.9440798858773182, + "macro avg": { + "precision": 0.9349447056108351, + "recall": 0.8769997176484361, + "f1-score": 0.9023669109980978, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.943247155320955, + "recall": 0.9440798858773182, + "f1-score": 0.9420878910462924, + "support": 224320.0 + }, + "cross-entropy": 0.13271224543526663, + "roc-auc": 0.9813876825616359, + "score": 0.9813876825616359 + }, + "val": { + "0": { + "precision": 0.9351403678606002, + "recall": 0.9809449853652709, + "f1-score": 0.9574951897848522, + "support": 16741.0 + }, + "1": { + "precision": 0.8762126503686457, + "recall": 0.6647041507212246, + "f1-score": 0.7559424171409441, + "support": 3397.0 + }, + "accuracy": 0.9275995630151952, + "macro avg": { + "precision": 0.9056765091146229, + "recall": 0.8228245680432478, + "f1-score": 0.8567188034628981, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9252000830100605, + "recall": 0.9275995630151952, + "f1-score": 0.9234960454472141, + "support": 20138.0 + }, + "cross-entropy": 0.17351272311805546, + "roc-auc": 0.9583121450834431, + "score": 0.9583121450834431 + }, + "test": { + "0": { + "precision": 0.9321849105974706, + "recall": 0.975507720392485, + "f1-score": 0.9533543950938487, + "support": 13147.0 + }, + "1": { + "precision": 0.8730784391013008, + "recall": 0.7036213468869124, + "f1-score": 0.7792436235708003, + "support": 3148.0 + }, + "accuracy": 0.9229825099723842, + "macro avg": { + "precision": 0.9026316748493857, + "recall": 0.8395645336396986, + "f1-score": 0.8662990093323245, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9207662439960627, + "recall": 0.9229825099723842, + "f1-score": 0.9197182669100772, + "support": 16295.0 + }, + "cross-entropy": 0.17696141048282132, + "roc-auc": 0.9626260391126089, + "score": 0.9626260391126089 + } + }, + "time": "0:02:48.403690", + "diversity": { + "train": { + "mean_centered_corr": 0.05291927233338356, + "mean_pairwise_disagreement": 0.03400679940235148, + "member_std": 0.03158120810985565 + }, + "val": { + "mean_centered_corr": 0.06288693100214005, + "mean_pairwise_disagreement": 0.03244768356405319, + "member_std": 0.0349886491894722 + }, + "test": { + "mean_centered_corr": 0.07999585568904877, + "mean_pairwise_disagreement": 0.041502439893495925, + "member_std": 0.0369756743311882 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml new file mode 100644 index 000000000..12aa82432 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml @@ -0,0 +1,34 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1/report.json new file mode 100644 index 000000000..a5a6399d1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1/report.json @@ -0,0 +1,258 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9421632002014817, + "recall": 0.9867781129960268, + "f1-score": 0.9639547024766547, + "support": 181971.0 + }, + "1": { + "precision": 0.9286730700818214, + "recall": 0.7397105008382725, + "f1-score": 0.8234907532761137, + "support": 42349.0 + }, + "accuracy": 0.9401346291012839, + "macro avg": { + "precision": 0.9354181351416515, + "recall": 0.8632443069171496, + "f1-score": 0.8937227278763842, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9396164209555941, + "recall": 0.9401346291012839, + "f1-score": 0.9374367469457449, + "support": 224320.0 + }, + "cross-entropy": 0.1388001827221654, + "roc-auc": 0.978892038769101, + "score": 0.978892038769101 + }, + "val": { + "0": { + "precision": 0.9301735798016231, + "recall": 0.9859028731855922, + "f1-score": 0.9572277801942873, + "support": 16741.0 + }, + "1": { + "precision": 0.9014202172096909, + "recall": 0.6352664115395937, + "f1-score": 0.7452944223795545, + "support": 3397.0 + }, + "accuracy": 0.9267553878240143, + "macro avg": { + "precision": 0.915796898505657, + "recall": 0.810584642362593, + "f1-score": 0.8512611012869209, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9253232881775892, + "recall": 0.9267553878240143, + "f1-score": 0.9214775757799141, + "support": 20138.0 + }, + "cross-entropy": 0.17373390370141575, + "roc-auc": 0.9586997364143323, + "score": 0.9586997364143323 + }, + "test": { + "0": { + "precision": 0.9266714593817398, + "recall": 0.9804518141020765, + "f1-score": 0.952803341094726, + "support": 13147.0 + }, + "1": { + "precision": 0.8922431865828092, + "recall": 0.6759847522236341, + "f1-score": 0.7692029640339779, + "support": 3148.0 + }, + "accuracy": 0.9216324025774778, + "macro avg": { + "precision": 0.9094573229822744, + "recall": 0.8282182831628553, + "f1-score": 0.861003152564352, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9200203269625293, + "recall": 0.9216324025774778, + "f1-score": 0.9173339340994984, + "support": 16295.0 + }, + "cross-entropy": 0.17809500689433352, + "roc-auc": 0.9622916084556132, + "score": 0.9622916084556132 + } + }, + "time": "0:02:44.189369", + "diversity": { + "train": { + "mean_centered_corr": 0.05150473117828369, + "mean_pairwise_disagreement": 0.03169338301274677, + "member_std": 0.03007516823709011 + }, + "val": { + "mean_centered_corr": 0.057541172951459885, + "mean_pairwise_disagreement": 0.02847489419777727, + "member_std": 0.031669627875089645 + }, + "test": { + "mean_centered_corr": 0.06821995228528976, + "mean_pairwise_disagreement": 0.035880044343703295, + "member_std": 0.033187758177518845 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml new file mode 100644 index 000000000..af2af4423 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml @@ -0,0 +1,34 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2/report.json new file mode 100644 index 000000000..7b3d57080 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2/report.json @@ -0,0 +1,258 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9470221716165355, + "recall": 0.9872562111545247, + "f1-score": 0.9667207464626163, + "support": 181971.0 + }, + "1": { + "precision": 0.9330117280027731, + "recall": 0.7626862499704834, + "f1-score": 0.8392947626905037, + "support": 42349.0 + }, + "accuracy": 0.9448600213980028, + "macro avg": { + "precision": 0.9400169498096542, + "recall": 0.874971230562504, + "f1-score": 0.90300775457656, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9443771632508113, + "recall": 0.9448600213980028, + "f1-score": 0.942664206756994, + "support": 224320.0 + }, + "cross-entropy": 0.1304695324130005, + "roc-auc": 0.9818820805222398, + "score": 0.9818820805222398 + }, + "val": { + "0": { + "precision": 0.9318168958297969, + "recall": 0.9836927304223164, + "f1-score": 0.9570523624106468, + "support": 16741.0 + }, + "1": { + "precision": 0.8892494929006085, + "recall": 0.6452752428613483, + "f1-score": 0.7478676219720232, + "support": 3397.0 + }, + "accuracy": 0.926606415731453, + "macro avg": { + "precision": 0.9105331943652026, + "recall": 0.8144839866418323, + "f1-score": 0.852459992191335, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9246363680837221, + "recall": 0.926606415731453, + "f1-score": 0.9217658114487834, + "support": 20138.0 + }, + "cross-entropy": 0.17503044041039684, + "roc-auc": 0.9582475934195426, + "score": 0.9582475934195426 + }, + "test": { + "0": { + "precision": 0.9284580989470648, + "recall": 0.9792348064197155, + "f1-score": 0.9531706955910118, + "support": 13147.0 + }, + "1": { + "precision": 0.8876080691642652, + "recall": 0.6848792884371029, + "f1-score": 0.7731755424063116, + "support": 3148.0 + }, + "accuracy": 0.9223688247928813, + "macro avg": { + "precision": 0.908033084055665, + "recall": 0.8320570474284092, + "f1-score": 0.8631731189986617, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9205663595327503, + "recall": 0.9223688247928813, + "f1-score": 0.9183977749266709, + "support": 16295.0 + }, + "cross-entropy": 0.17851016154157287, + "roc-auc": 0.9625649084455907, + "score": 0.9625649084455907 + } + }, + "time": "0:02:48.224028", + "diversity": { + "train": { + "mean_centered_corr": 0.05968523025512695, + "mean_pairwise_disagreement": 0.033705988852330775, + "member_std": 0.031936075538396835 + }, + "val": { + "mean_centered_corr": 0.06932749599218369, + "mean_pairwise_disagreement": 0.031030346255994927, + "member_std": 0.03422032669186592 + }, + "test": { + "mean_centered_corr": 0.08440784364938736, + "mean_pairwise_disagreement": 0.03857926931871047, + "member_std": 0.03628651052713394 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0b339388c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9453926712602988, + "recall": 0.9862120887394146, + "f1-score": 0.9653710741556594, + "support": 181971.0 + }, + "1": { + "precision": 0.9272584947234142, + "recall": 0.7552244444969185, + "f1-score": 0.8324462201168646, + "support": 42349.0 + }, + "accuracy": 0.9426043152639088, + "macro avg": { + "precision": 0.9363255829918564, + "recall": 0.8707182666181665, + "f1-score": 0.898908647136262, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9419691502092978, + "recall": 0.9426043152639088, + "f1-score": 0.9402764118710262, + "support": 224320.0 + }, + "cross-entropy": 0.13432360872538882, + "roc-auc": 0.9807020207732833, + "score": 0.9807020207732833 + }, + "val": { + "0": { + "precision": 0.9332728098048116, + "recall": 0.9824980586583836, + "f1-score": 0.9572530190600902, + "support": 16741.0 + }, + "1": { + "precision": 0.8834526650755767, + "recall": 0.6538121872240212, + "f1-score": 0.7514802909829132, + "support": 3397.0 + }, + "accuracy": 0.927053332009137, + "macro avg": { + "precision": 0.9083627374401941, + "recall": 0.8181551229412024, + "f1-score": 0.8543666550215017, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9248688455757318, + "recall": 0.927053332009137, + "f1-score": 0.9225420270411127, + "support": 20138.0 + }, + "cross-entropy": 0.17301902153958362, + "roc-auc": 0.9589474980445032, + "score": 0.9589474980445032 + }, + "test": { + "0": { + "precision": 0.9310144506571781, + "recall": 0.9752034684718948, + "f1-score": 0.952596775391931, + "support": 13147.0 + }, + "1": { + "precision": 0.8708399366085579, + "recall": 0.6982210927573063, + "f1-score": 0.7750352609308886, + "support": 3148.0 + }, + "accuracy": 0.921693771095428, + "macro avg": { + "precision": 0.900927193632868, + "recall": 0.8367122806146006, + "f1-score": 0.8638160181614098, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9193894509502093, + "recall": 0.921693771095428, + "f1-score": 0.9182940047553332, + "support": 16295.0 + }, + "cross-entropy": 0.17739591453615167, + "roc-auc": 0.9625170670540112, + "score": 0.9625170670540112 + } + }, + "time": "0:02:48.403690", + "diversity": { + "train": { + "mean_centered_corr": 0.05291927233338356, + "mean_pairwise_disagreement": 0.03400679940235148, + "member_std": 0.03158120810985565 + }, + "val": { + "mean_centered_corr": 0.06288693100214005, + "mean_pairwise_disagreement": 0.03244768356405319, + "member_std": 0.0349886491894722 + }, + "test": { + "mean_centered_corr": 0.07999585568904877, + "mean_pairwise_disagreement": 0.041502439893495925, + "member_std": 0.0369756743311882 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 6, + 9, + 15, + 28, + 3, + 23, + 26, + 2, + 20, + 13, + 6, + 15, + 5 + ], + "head_selection_time": "0:00:13.989049" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..7a16f3ac1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1/report.json @@ -0,0 +1,277 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9438133041335889, + "recall": 0.9852284155167582, + "f1-score": 0.9640762837868722, + "support": 181971.0 + }, + "1": { + "precision": 0.921778605517402, + "recall": 0.7479751587994994, + "f1-score": 0.8258313454042991, + "support": 42349.0 + }, + "accuracy": 0.9404377674750357, + "macro avg": { + "precision": 0.9327959548254954, + "recall": 0.8666017871581289, + "f1-score": 0.8949538145955857, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9396534100015593, + "recall": 0.9404377674750357, + "f1-score": 0.9379772516204866, + "support": 224320.0 + }, + "cross-entropy": 0.138611871152865, + "roc-auc": 0.9786723684054657, + "score": 0.9786723684054657 + }, + "val": { + "0": { + "precision": 0.9314650545228544, + "recall": 0.984767935009856, + "f1-score": 0.9573751451800232, + "support": 16741.0 + }, + "1": { + "precision": 0.8954489544895449, + "recall": 0.6429202237268178, + "f1-score": 0.748457847840987, + "support": 3397.0 + }, + "accuracy": 0.927102989373324, + "macro avg": { + "precision": 0.9134570045061996, + "recall": 0.8138440793683369, + "f1-score": 0.8529164965105052, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9253896402904007, + "recall": 0.927102989373324, + "f1-score": 0.9221337081425465, + "support": 20138.0 + }, + "cross-entropy": 0.17287177515474142, + "roc-auc": 0.9593694489371631, + "score": 0.9593694489371631 + }, + "test": { + "0": { + "precision": 0.9287417792874177, + "recall": 0.9774853578763216, + "f1-score": 0.9524903646605396, + "support": 13147.0 + }, + "1": { + "precision": 0.8795768917819365, + "recall": 0.6867852604828463, + "f1-score": 0.7713164466642882, + "support": 3148.0 + }, + "accuracy": 0.9213255599877263, + "macro avg": { + "precision": 0.9041593355346771, + "recall": 0.832135309179584, + "f1-score": 0.861903405662414, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9192437083535575, + "recall": 0.9213255599877263, + "f1-score": 0.9174897206683826, + "support": 16295.0 + }, + "cross-entropy": 0.17752804180302853, + "roc-auc": 0.9625145058481993, + "score": 0.9625145058481993 + } + }, + "time": "0:02:44.189369", + "diversity": { + "train": { + "mean_centered_corr": 0.05150473117828369, + "mean_pairwise_disagreement": 0.03169338301274677, + "member_std": 0.03007516823709011 + }, + "val": { + "mean_centered_corr": 0.057541172951459885, + "mean_pairwise_disagreement": 0.02847489419777727, + "member_std": 0.031669627875089645 + }, + "test": { + "mean_centered_corr": 0.06821995228528976, + "mean_pairwise_disagreement": 0.035880044343703295, + "member_std": 0.033187758177518845 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 14, + 26, + 8, + 24, + 18, + 6, + 17, + 9, + 30, + 14, + 5, + 10, + 15, + 26, + 28 + ], + "head_selection_time": "0:00:15.746127" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..bd1780ddf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2/report.json @@ -0,0 +1,269 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9458610786757976, + "recall": 0.9882288936149166, + "f1-score": 0.9665809354113085, + "support": 181971.0 + }, + "1": { + "precision": 0.9373647581729926, + "recall": 0.7569482160145458, + "f1-score": 0.8375507857917358, + "support": 42349.0 + }, + "accuracy": 0.9445657988587732, + "macro avg": { + "precision": 0.9416129184243951, + "recall": 0.8725885548147312, + "f1-score": 0.9020658606015222, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9442570724482062, + "recall": 0.9445657988587732, + "f1-score": 0.942221547901326, + "support": 224320.0 + }, + "cross-entropy": 0.1303272598008576, + "roc-auc": 0.9816630888898851, + "score": 0.9816630888898851 + }, + "val": { + "0": { + "precision": 0.9299154929577464, + "recall": 0.9859626067737889, + "f1-score": 0.9571192484996086, + "support": 16741.0 + }, + "1": { + "precision": 0.9015912897822446, + "recall": 0.6337945245805122, + "f1-score": 0.7443388072601556, + "support": 3397.0 + }, + "accuracy": 0.9265567583672659, + "macro avg": { + "precision": 0.9157533913699956, + "recall": 0.8098785656771506, + "f1-score": 0.850729027879882, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9251375945474187, + "recall": 0.9265567583672659, + "f1-score": 0.921226152914624, + "support": 20138.0 + }, + "cross-entropy": 0.1753619698028255, + "roc-auc": 0.9591019138539669, + "score": 0.9591019138539669 + }, + "test": { + "0": { + "precision": 0.9280829313944281, + "recall": 0.9806039400623716, + "f1-score": 0.9536208299430431, + "support": 13147.0 + }, + "1": { + "precision": 0.8939267886855241, + "recall": 0.6826556543837357, + "f1-score": 0.774135446685879, + "support": 3148.0 + }, + "accuracy": 0.9230438784903344, + "macro avg": { + "precision": 0.911004860039976, + "recall": 0.8316297972230536, + "f1-score": 0.863878138314461, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9214843712687681, + "recall": 0.9230438784903344, + "f1-score": 0.9189463907596401, + "support": 16295.0 + }, + "cross-entropy": 0.17871078362700502, + "roc-auc": 0.9629950822915426, + "score": 0.9629950822915426 + } + }, + "time": "0:02:48.224028", + "diversity": { + "train": { + "mean_centered_corr": 0.05968523025512695, + "mean_pairwise_disagreement": 0.033705988852330775, + "member_std": 0.031936075538396835 + }, + "val": { + "mean_centered_corr": 0.06932749599218369, + "mean_pairwise_disagreement": 0.031030346255994927, + "member_std": 0.03422032669186592 + }, + "test": { + "mean_centered_corr": 0.08440784364938736, + "mean_pairwise_disagreement": 0.03857926931871047, + "member_std": 0.03628651052713394 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1, + 6, + 25, + 11, + 5, + 18, + 4, + 14 + ], + "head_selection_time": "0:00:09.905004" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a89b4d488 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/0/report.json @@ -0,0 +1,621 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2420, + "metrics": { + "train": { + "0": { + "precision": 0.9511390416339356, + "recall": 0.9648020838485253, + "f1-score": 0.9579218455024607, + "support": 181971.0 + }, + "1": { + "precision": 0.8388070970177426, + "recall": 0.7870315709934119, + "f1-score": 0.8120949271477998, + "support": 42349.0 + }, + "accuracy": 0.9312410841654779, + "macro avg": { + "precision": 0.8949730693258391, + "recall": 0.8759168274209685, + "f1-score": 0.8850083863251303, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9299320804956013, + "recall": 0.9312410841654779, + "f1-score": 0.9303914239377251, + "support": 224320.0 + }, + "cross-entropy": 0.15615645016500299, + "roc-auc": 0.9724991426058968, + "score": 0.9724991426058968 + }, + "val": { + "0": { + "precision": 0.9410138248847926, + "recall": 0.9758078967803596, + "f1-score": 0.958095070525791, + "support": 16741.0 + }, + "1": { + "precision": 0.8542116630669546, + "recall": 0.6985575507801001, + "f1-score": 0.768582995951417, + "support": 3397.0 + }, + "accuracy": 0.9290396265766213, + "macro avg": { + "precision": 0.8976127439758737, + "recall": 0.8371827237802298, + "f1-score": 0.863339033238604, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9263715096749804, + "recall": 0.9290396265766213, + "f1-score": 0.9261270241791256, + "support": 20138.0 + }, + "cross-entropy": 0.17528768630111824, + "roc-auc": 0.957813544584969, + "score": 0.957813544584969 + }, + "test": { + "0": { + "precision": 0.9342733348020856, + "recall": 0.9676732334372861, + "f1-score": 0.9506800179345389, + "support": 13147.0 + }, + "1": { + "precision": 0.8412994772218073, + "recall": 0.71569250317662, + "f1-score": 0.7734294541709578, + "support": 3148.0 + }, + "accuracy": 0.9189935563056152, + "macro avg": { + "precision": 0.8877864060119465, + "recall": 0.841682868306953, + "f1-score": 0.8620547360527484, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9163118924171383, + "recall": 0.9189935563056152, + "f1-score": 0.9164373192706695, + "support": 16295.0 + }, + "cross-entropy": 0.1803789882797406, + "roc-auc": 0.961490313471295, + "score": 0.961490313471295 + } + }, + "time": "0:03:46.650441", + "diversity": { + "train": { + "mean_centered_corr": 0.0506281740963459, + "mean_pairwise_disagreement": 0.03327413710540242, + "member_std": 0.03079444356262684 + }, + "val": { + "mean_centered_corr": 0.056434109807014465, + "mean_pairwise_disagreement": 0.030276074921749604, + "member_std": 0.03229342773556709 + }, + "test": { + "mean_centered_corr": 0.06734751909971237, + "mean_pairwise_disagreement": 0.037917825574834946, + "member_std": 0.03452758118510246 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.860463" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..008857947 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/1/report.json @@ -0,0 +1,621 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9352279269377715, + "recall": 0.9819916360299169, + "f1-score": 0.9580394648309435, + "support": 181971.0 + }, + "1": { + "precision": 0.9014436090225564, + "recall": 0.7077616944910151, + "f1-score": 0.7929469966533949, + "support": 42349.0 + }, + "accuracy": 0.9302202211126962, + "macro avg": { + "precision": 0.9183357679801639, + "recall": 0.844876665260466, + "f1-score": 0.8754932307421692, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9288498417051063, + "recall": 0.9302202211126962, + "f1-score": 0.9268719321327845, + "support": 224320.0 + }, + "cross-entropy": 0.16027100539450811, + "roc-auc": 0.9698985230711175, + "score": 0.9698985230711175 + }, + "val": { + "0": { + "precision": 0.9274849862490879, + "recall": 0.9870975449495251, + "f1-score": 0.9563632154638578, + "support": 16741.0 + }, + "1": { + "precision": 0.9069366652305041, + "recall": 0.6196644097733294, + "f1-score": 0.7362714235746765, + "support": 3397.0 + }, + "accuracy": 0.9251166948058397, + "macro avg": { + "precision": 0.917210825739796, + "recall": 0.8033809773614272, + "f1-score": 0.8463173195192671, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9240187708106069, + "recall": 0.9251166948058397, + "f1-score": 0.9192367968995738, + "support": 20138.0 + }, + "cross-entropy": 0.18037389209877025, + "roc-auc": 0.9580043948939159, + "score": 0.9580043948939159 + }, + "test": { + "0": { + "precision": 0.9213914775556662, + "recall": 0.9851677188712253, + "f1-score": 0.9522129098661961, + "support": 13147.0 + }, + "1": { + "precision": 0.9128686327077749, + "recall": 0.6489834815756036, + "f1-score": 0.7586334942443371, + "support": 3148.0 + }, + "accuracy": 0.9202209266646211, + "macro avg": { + "precision": 0.9171300551317205, + "recall": 0.8170756002234145, + "f1-score": 0.8554232020552666, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9197449653997188, + "recall": 0.9202209266646211, + "f1-score": 0.9148156714263305, + "support": 16295.0 + }, + "cross-entropy": 0.18507285757654685, + "roc-auc": 0.9620360194454477, + "score": 0.9620360194454477 + } + }, + "time": "0:03:27.053329", + "diversity": { + "train": { + "mean_centered_corr": 0.047042831778526306, + "mean_pairwise_disagreement": 0.027546063946666054, + "member_std": 0.02635623700916767 + }, + "val": { + "mean_centered_corr": 0.04908687248826027, + "mean_pairwise_disagreement": 0.024034264382214333, + "member_std": 0.02737056463956833 + }, + "test": { + "mean_centered_corr": 0.05689273402094841, + "mean_pairwise_disagreement": 0.0295994219481535, + "member_std": 0.02851640060544014 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24 + ], + "head_selection_time": "0:00:01.177794" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..e420fe840 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-best-head-evaluation/2/report.json @@ -0,0 +1,621 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9298072451036122, + "recall": 0.9887674409658682, + "f1-score": 0.9583813828129786, + "support": 181971.0 + }, + "1": { + "precision": 0.9336579032781565, + "recall": 0.6792604311790125, + "f1-score": 0.7863967522792821, + "support": 42349.0 + }, + "accuracy": 0.9303361269614836, + "macro avg": { + "precision": 0.9317325741908844, + "recall": 0.8340139360724403, + "f1-score": 0.8723890675461303, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9305342044609312, + "recall": 0.9303361269614836, + "f1-score": 0.9259126902377668, + "support": 224320.0 + }, + "cross-entropy": 0.15872104617363245, + "roc-auc": 0.9706236144688893, + "score": 0.9706236144688893 + }, + "val": { + "0": { + "precision": 0.9251312995865459, + "recall": 0.9890687533600143, + "f1-score": 0.9560322180201507, + "support": 16741.0 + }, + "1": { + "precision": 0.9183035714285714, + "recall": 0.6055342949661466, + "f1-score": 0.7298208266808586, + "support": 3397.0 + }, + "accuracy": 0.924371834343033, + "macro avg": { + "precision": 0.9217174355075587, + "recall": 0.7973015241630805, + "f1-score": 0.8429265223505047, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9239795569828794, + "recall": 0.924371834343033, + "f1-score": 0.917873508298253, + "support": 20138.0 + }, + "cross-entropy": 0.18042645510918942, + "roc-auc": 0.9575815999587967, + "score": 0.9575815999587967 + }, + "test": { + "0": { + "precision": 0.9182661661590764, + "recall": 0.9861565376131437, + "f1-score": 0.9510012469742537, + "support": 13147.0 + }, + "1": { + "precision": 0.9163602941176471, + "recall": 0.633418043202033, + "f1-score": 0.749060856498873, + "support": 3148.0 + }, + "accuracy": 0.9180116600184105, + "macro avg": { + "precision": 0.9173132301383617, + "recall": 0.8097872904075883, + "f1-score": 0.8500310517365633, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9178979743710175, + "recall": 0.9180116600184105, + "f1-score": 0.9119887677342108, + "support": 16295.0 + }, + "cross-entropy": 0.18508615173756227, + "roc-auc": 0.9611081622343147, + "score": 0.9611081622343147 + } + }, + "time": "0:03:33.015302", + "diversity": { + "train": { + "mean_centered_corr": 0.02876308374106884, + "mean_pairwise_disagreement": 0.026535222219985274, + "member_std": 0.025292295962572098 + }, + "val": { + "mean_centered_corr": 0.041079021990299225, + "mean_pairwise_disagreement": 0.02388218870439131, + "member_std": 0.026205172762274742 + }, + "test": { + "mean_centered_corr": 0.053323205560445786, + "mean_pairwise_disagreement": 0.030486048560314366, + "member_std": 0.028108706697821617 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27 + ], + "head_selection_time": "0:00:01.182248" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..e655e5952 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml @@ -0,0 +1,45 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..5caf5c656 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,617 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2420, + "metrics": { + "train": { + "0": { + "precision": 0.9480090078156047, + "recall": 0.9831951244978595, + "f1-score": 0.965281524483674, + "support": 181971.0 + }, + "1": { + "precision": 0.9140890574518893, + "recall": 0.7683062173841176, + "f1-score": 0.8348814533511238, + "support": 42349.0 + }, + "accuracy": 0.942626604850214, + "macro avg": { + "precision": 0.931049032633747, + "recall": 0.8757506709409886, + "f1-score": 0.9000814889173989, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9416053167583962, + "recall": 0.942626604850214, + "f1-score": 0.9406635117679449, + "support": 224320.0 + }, + "cross-entropy": 0.13436786219631347, + "roc-auc": 0.9804692077039372, + "score": 0.9804692077039372 + }, + "val": { + "0": { + "precision": 0.9353075170842825, + "recall": 0.9810644525416642, + "f1-score": 0.957639718958631, + "support": 16741.0 + }, + "1": { + "precision": 0.8770364623739333, + "recall": 0.6655872828966736, + "f1-score": 0.7568200836820084, + "support": 3397.0 + }, + "accuracy": 0.9278478498361307, + "macro avg": { + "precision": 0.9061719897291078, + "recall": 0.8233258677191688, + "f1-score": 0.8572299013203197, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9254780020951546, + "recall": 0.9278478498361307, + "f1-score": 0.9237642446794232, + "support": 20138.0 + }, + "cross-entropy": 0.1724175473381701, + "roc-auc": 0.9589599740471012, + "score": 0.9589599740471012 + }, + "test": { + "0": { + "precision": 0.931104503589818, + "recall": 0.9765726021145509, + "f1-score": 0.9532967032967034, + "support": 13147.0 + }, + "1": { + "precision": 0.8770949720670391, + "recall": 0.6982210927573063, + "f1-score": 0.7775026529890343, + "support": 3148.0 + }, + "accuracy": 0.9227984044185333, + "macro avg": { + "precision": 0.9040997378284286, + "recall": 0.8373968474359286, + "f1-score": 0.8653996781428688, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9206705051096887, + "recall": 0.9227984044185333, + "f1-score": 0.919335385692006, + "support": 16295.0 + }, + "cross-entropy": 0.17668487055650292, + "roc-auc": 0.9628908822909436, + "score": 0.9628908822909436 + } + }, + "time": "0:03:46.650441", + "diversity": { + "train": { + "mean_centered_corr": 0.0506281740963459, + "mean_pairwise_disagreement": 0.03327413710540242, + "member_std": 0.03079444356262684 + }, + "val": { + "mean_centered_corr": 0.056434109807014465, + "mean_pairwise_disagreement": 0.030276074921749604, + "member_std": 0.03229342773556709 + }, + "test": { + "mean_centered_corr": 0.06734751909971237, + "mean_pairwise_disagreement": 0.037917825574834946, + "member_std": 0.03452758118510246 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml new file mode 100644 index 000000000..138dbeb9f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml @@ -0,0 +1,45 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1/report.json new file mode 100644 index 000000000..9594849bb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1/report.json @@ -0,0 +1,617 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9375189502849077, + "recall": 0.9855306614790269, + "f1-score": 0.9609254650231339, + "support": 181971.0 + }, + "1": { + "precision": 0.9202845897668785, + "recall": 0.71777373727833, + "f1-score": 0.8065110972552038, + "support": 42349.0 + }, + "accuracy": 0.9349812767475035, + "macro avg": { + "precision": 0.9289017700258931, + "recall": 0.8516521993786785, + "f1-score": 0.8837182811391688, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9342653040046919, + "recall": 0.9349812767475035, + "f1-score": 0.9317738331552484, + "support": 224320.0 + }, + "cross-entropy": 0.14907449987993657, + "roc-auc": 0.9745732258873414, + "score": 0.9745732258873414 + }, + "val": { + "0": { + "precision": 0.9290293555280621, + "recall": 0.9867988770085419, + "f1-score": 0.9570431306665121, + "support": 16741.0 + }, + "1": { + "precision": 0.9061969439728353, + "recall": 0.6284957315278187, + "f1-score": 0.7422214496784286, + "support": 3397.0 + }, + "accuracy": 0.9263581289105174, + "macro avg": { + "precision": 0.9176131497504487, + "recall": 0.8076473042681802, + "f1-score": 0.8496322901724704, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9251778458422389, + "recall": 0.9263581289105174, + "f1-score": 0.9208057063782751, + "support": 20138.0 + }, + "cross-entropy": 0.17400536493244298, + "roc-auc": 0.9589376508824807, + "score": 0.9589376508824807 + }, + "test": { + "0": { + "precision": 0.9259578131726216, + "recall": 0.9816688217844375, + "f1-score": 0.9529998153959757, + "support": 13147.0 + }, + "1": { + "precision": 0.8977513788714467, + "recall": 0.6721728081321474, + "f1-score": 0.768755676657584, + "support": 3148.0 + }, + "accuracy": 0.9218778766492789, + "macro avg": { + "precision": 0.9118545960220341, + "recall": 0.8269208149582925, + "f1-score": 0.8608777460267798, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9205086658771261, + "recall": 0.9218778766492789, + "f1-score": 0.9174060413089272, + "support": 16295.0 + }, + "cross-entropy": 0.17781928920731452, + "roc-auc": 0.9627469836002609, + "score": 0.9627469836002609 + } + }, + "time": "0:03:27.053329", + "diversity": { + "train": { + "mean_centered_corr": 0.047042831778526306, + "mean_pairwise_disagreement": 0.027546063946666054, + "member_std": 0.02635623700916767 + }, + "val": { + "mean_centered_corr": 0.04908687248826027, + "mean_pairwise_disagreement": 0.024034264382214333, + "member_std": 0.02737056463956833 + }, + "test": { + "mean_centered_corr": 0.05689273402094841, + "mean_pairwise_disagreement": 0.0295994219481535, + "member_std": 0.02851640060544014 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml new file mode 100644 index 000000000..7cd9bf24e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml @@ -0,0 +1,45 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2/report.json new file mode 100644 index 000000000..4a6d2f99f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2/report.json @@ -0,0 +1,617 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.938750961020486, + "recall": 0.9863769501733792, + "f1-score": 0.9619748428347098, + "support": 181971.0 + }, + "1": { + "precision": 0.9251441857656189, + "recall": 0.7234645446173463, + "f1-score": 0.8119683036069223, + "support": 42349.0 + }, + "accuracy": 0.9367421540656206, + "macro avg": { + "precision": 0.9319475733930525, + "recall": 0.8549207473953628, + "f1-score": 0.8869715732208161, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9361821605333767, + "recall": 0.9367421540656206, + "f1-score": 0.9336553575914966, + "support": 224320.0 + }, + "cross-entropy": 0.14475981504575822, + "roc-auc": 0.9761314634035192, + "score": 0.9761314634035192 + }, + "val": { + "0": { + "precision": 0.9294680551646496, + "recall": 0.9863210083029688, + "f1-score": 0.9570509476612763, + "support": 16741.0 + }, + "1": { + "precision": 0.9034976822587442, + "recall": 0.6311451280541654, + "f1-score": 0.7431542461005199, + "support": 3397.0 + }, + "accuracy": 0.9264077862747045, + "macro avg": { + "precision": 0.9164828687116968, + "recall": 0.8087330681785672, + "f1-score": 0.8501025968808982, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9250872151228698, + "recall": 0.9264077862747045, + "f1-score": 0.9209695545138987, + "support": 20138.0 + }, + "cross-entropy": 0.17454952390147907, + "roc-auc": 0.9589488871977169, + "score": 0.9589488871977169 + }, + "test": { + "0": { + "precision": 0.9246472315736695, + "recall": 0.9818970107248802, + "f1-score": 0.9524125719344843, + "support": 13147.0 + }, + "1": { + "precision": 0.8980291345329906, + "recall": 0.6658195679796697, + "f1-score": 0.7646844217438891, + "support": 3148.0 + }, + "accuracy": 0.920834611844124, + "macro avg": { + "precision": 0.91133818305333, + "recall": 0.823858289352275, + "f1-score": 0.8585484968391867, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9195049321269646, + "recall": 0.920834611844124, + "f1-score": 0.9161457283137421, + "support": 16295.0 + }, + "cross-entropy": 0.1789803337967533, + "roc-auc": 0.9624131449200803, + "score": 0.9624131449200803 + } + }, + "time": "0:03:33.015302", + "diversity": { + "train": { + "mean_centered_corr": 0.02876308374106884, + "mean_pairwise_disagreement": 0.026535222219985274, + "member_std": 0.025292295962572098 + }, + "val": { + "mean_centered_corr": 0.041079021990299225, + "mean_pairwise_disagreement": 0.02388218870439131, + "member_std": 0.026205172762274742 + }, + "test": { + "mean_centered_corr": 0.053323205560445786, + "mean_pairwise_disagreement": 0.030486048560314366, + "member_std": 0.028108706697821617 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..a6950ee69 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,639 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2420, + "metrics": { + "train": { + "0": { + "precision": 0.9470743368373016, + "recall": 0.9808101290865028, + "f1-score": 0.9636470639051465, + "support": 181971.0 + }, + "1": { + "precision": 0.9026403100342933, + "recall": 0.7644808614134927, + "f1-score": 0.8278357369336197, + "support": 42349.0 + }, + "accuracy": 0.9399696861626248, + "macro avg": { + "precision": 0.9248573234357975, + "recall": 0.8726454952499978, + "f1-score": 0.8957414004193831, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9386857107625842, + "recall": 0.9399696861626248, + "f1-score": 0.9380074691926055, + "support": 224320.0 + }, + "cross-entropy": 0.13890143673129085, + "roc-auc": 0.9784781707405066, + "score": 0.9784781707405066 + }, + "val": { + "0": { + "precision": 0.9363594890510949, + "recall": 0.9808255181888776, + "f1-score": 0.9580768445313184, + "support": 16741.0 + }, + "1": { + "precision": 0.8766333589546502, + "recall": 0.6714748307329997, + "f1-score": 0.7604600766794466, + "support": 3397.0 + }, + "accuracy": 0.9286423676631245, + "macro avg": { + "precision": 0.9064964240028726, + "recall": 0.8261501744609387, + "f1-score": 0.8592684606053824, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9262845231191442, + "recall": 0.9286423676631245, + "f1-score": 0.9247416493583713, + "support": 20138.0 + }, + "cross-entropy": 0.17179269209939912, + "roc-auc": 0.9594844321379927, + "score": 0.9594844321379927 + }, + "test": { + "0": { + "precision": 0.9321799811005307, + "recall": 0.9754316574123374, + "f1-score": 0.9533154921201308, + "support": 13147.0 + }, + "1": { + "precision": 0.8727344365642238, + "recall": 0.7036213468869124, + "f1-score": 0.7791065775589167, + "support": 3148.0 + }, + "accuracy": 0.9229211414544338, + "macro avg": { + "precision": 0.9024572088323772, + "recall": 0.8395265021496249, + "f1-score": 0.8662110348395238, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9206958096245998, + "recall": 0.9229211414544338, + "f1-score": 0.9196604038698268, + "support": 16295.0 + }, + "cross-entropy": 0.1758390088286691, + "roc-auc": 0.9634382071404677, + "score": 0.9634382071404677 + } + }, + "time": "0:03:46.650441", + "diversity": { + "train": { + "mean_centered_corr": 0.0506281740963459, + "mean_pairwise_disagreement": 0.03327413710540242, + "member_std": 0.03079444356262684 + }, + "val": { + "mean_centered_corr": 0.056434109807014465, + "mean_pairwise_disagreement": 0.030276074921749604, + "member_std": 0.03229342773556709 + }, + "test": { + "mean_centered_corr": 0.06734751909971237, + "mean_pairwise_disagreement": 0.037917825574834946, + "member_std": 0.03452758118510246 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 2, + 0, + 28, + 7, + 26, + 18, + 29, + 24, + 15, + 4, + 7, + 27, + 19, + 31, + 0, + 2, + 28, + 11 + ], + "head_selection_time": "0:00:17.095036" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..83cdf249d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,631 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9325960268331218, + "recall": 0.989349951365877, + "f1-score": 0.9601350342385392, + "support": 181971.0 + }, + "1": { + "precision": 0.9380335731414868, + "recall": 0.6927436303100427, + "f1-score": 0.7969412148212539, + "support": 42349.0 + }, + "accuracy": 0.9333541369472182, + "macro avg": { + "precision": 0.9353147999873044, + "recall": 0.8410467908379599, + "f1-score": 0.8785381245298965, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9336225721639569, + "recall": 0.9333541369472182, + "f1-score": 0.929325944284444, + "support": 224320.0 + }, + "cross-entropy": 0.15036636948759663, + "roc-auc": 0.9745291516953071, + "score": 0.9745291516953071 + }, + "val": { + "0": { + "precision": 0.9258700631249651, + "recall": 0.9900244907711606, + "f1-score": 0.9568731597482825, + "support": 16741.0 + }, + "1": { + "precision": 0.9253464461332142, + "recall": 0.6093612010597587, + "f1-score": 0.7348242811501597, + "support": 3397.0 + }, + "accuracy": 0.9258118979044593, + "macro avg": { + "precision": 0.9256082546290896, + "recall": 0.7996928459154596, + "f1-score": 0.8458487204492211, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9257817362344607, + "recall": 0.9258118979044593, + "f1-score": 0.9194166079259654, + "support": 20138.0 + }, + "cross-entropy": 0.17630521274856525, + "roc-auc": 0.9596519921503349, + "score": 0.9596519921503349 + }, + "test": { + "0": { + "precision": 0.921048891415577, + "recall": 0.9858522856925535, + "f1-score": 0.9523494617730262, + "support": 13147.0 + }, + "1": { + "precision": 0.9163292847503374, + "recall": 0.6470775095298602, + "f1-score": 0.7585179668590579, + "support": 3148.0 + }, + "accuracy": 0.9204050322184719, + "macro avg": { + "precision": 0.9186890880829572, + "recall": 0.8164648976112068, + "f1-score": 0.855433714316042, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9201371195970944, + "recall": 0.9204050322184719, + "f1-score": 0.914903524615053, + "support": 16295.0 + }, + "cross-entropy": 0.18045087821080671, + "roc-auc": 0.9627588835423583, + "score": 0.9627588835423583 + } + }, + "time": "0:03:27.053329", + "diversity": { + "train": { + "mean_centered_corr": 0.047042831778526306, + "mean_pairwise_disagreement": 0.027546063946666054, + "member_std": 0.02635623700916767 + }, + "val": { + "mean_centered_corr": 0.04908687248826027, + "mean_pairwise_disagreement": 0.024034264382214333, + "member_std": 0.02737056463956833 + }, + "test": { + "mean_centered_corr": 0.05689273402094841, + "mean_pairwise_disagreement": 0.0295994219481535, + "member_std": 0.02851640060544014 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24, + 21, + 7, + 15, + 26, + 6, + 18, + 12, + 4, + 6, + 8 + ], + "head_selection_time": "0:00:18.810547" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..f829313d1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,626 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9369411248537522, + "recall": 0.985777953629974, + "f1-score": 0.9607393145112699, + "support": 181971.0 + }, + "1": { + "precision": 0.9212512171372931, + "recall": 0.7149165269545916, + "f1-score": 0.8050735910015556, + "support": 42349.0 + }, + "accuracy": 0.9346424750356633, + "macro avg": { + "precision": 0.9290961709955227, + "recall": 0.8503472402922828, + "f1-score": 0.8829064527564128, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9339790532512009, + "recall": 0.9346424750356633, + "f1-score": 0.9313514412725357, + "support": 224320.0 + }, + "cross-entropy": 0.14923608977800548, + "roc-auc": 0.9741342548580761, + "score": 0.9741342548580761 + }, + "val": { + "0": { + "precision": 0.929322462749508, + "recall": 0.9872767457141151, + "f1-score": 0.9574233910676012, + "support": 16741.0 + }, + "1": { + "precision": 0.9094772630684232, + "recall": 0.6299676184869002, + "f1-score": 0.7443478260869565, + "support": 3397.0 + }, + "accuracy": 0.9270036746449498, + "macro avg": { + "precision": 0.9193998629089657, + "recall": 0.8086221821005076, + "f1-score": 0.8508856085772789, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9259748540834715, + "recall": 0.9270036746449498, + "f1-score": 0.9214805122196894, + "support": 20138.0 + }, + "cross-entropy": 0.174415932178035, + "roc-auc": 0.9594475404488445, + "score": 0.9594475404488445 + }, + "test": { + "0": { + "precision": 0.9254373386865501, + "recall": 0.9818209477447326, + "f1-score": 0.9527957187673003, + "support": 13147.0 + }, + "1": { + "precision": 0.898167873881551, + "recall": 0.6696315120711563, + "f1-score": 0.7672429481346679, + "support": 3148.0 + }, + "accuracy": 0.9215096655415772, + "macro avg": { + "precision": 0.9118026062840505, + "recall": 0.8257262299079444, + "f1-score": 0.860019333450984, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9201692027426325, + "recall": 0.9215096655415772, + "f1-score": 0.9169491325781916, + "support": 16295.0 + }, + "cross-entropy": 0.17907506203888027, + "roc-auc": 0.9625258379758007, + "score": 0.9625258379758007 + } + }, + "time": "0:03:33.015302", + "diversity": { + "train": { + "mean_centered_corr": 0.02876308374106884, + "mean_pairwise_disagreement": 0.026535222219985274, + "member_std": 0.025292295962572098 + }, + "val": { + "mean_centered_corr": 0.041079021990299225, + "mean_pairwise_disagreement": 0.02388218870439131, + "member_std": 0.026205172762274742 + }, + "test": { + "mean_centered_corr": 0.053323205560445786, + "mean_pairwise_disagreement": 0.030486048560314366, + "member_std": 0.028108706697821617 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27, + 5, + 16, + 28, + 4, + 6 + ], + "head_selection_time": "0:00:10.772881" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4003e9150 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/0/report.json @@ -0,0 +1,657 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9432566752157673, + "recall": 0.9741662132977232, + "f1-score": 0.9584623078898636, + "support": 181971.0 + }, + "1": { + "precision": 0.8708019567965701, + "recall": 0.7481876785756453, + "f1-score": 0.8048517177875151, + "support": 42349.0 + }, + "accuracy": 0.9315041012838802, + "macro avg": { + "precision": 0.9070293160061687, + "recall": 0.8611769459366843, + "f1-score": 0.8816570128386894, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9295780693387408, + "recall": 0.9315041012838802, + "f1-score": 0.9294624198716558, + "support": 224320.0 + }, + "cross-entropy": 0.15818911070182506, + "roc-auc": 0.9710347146934778, + "score": 0.9710347146934778 + }, + "val": { + "0": { + "precision": 0.9363334478415206, + "recall": 0.9768831013678991, + "f1-score": 0.9561785599438711, + "support": 16741.0 + }, + "1": { + "precision": 0.8551646706586826, + "recall": 0.672652340300265, + "f1-score": 0.753007085187016, + "support": 3397.0 + }, + "accuracy": 0.9255636110835237, + "macro avg": { + "precision": 0.8957490592501016, + "recall": 0.824767720834082, + "f1-score": 0.8545928225654436, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9226414061248607, + "recall": 0.9255636110835237, + "f1-score": 0.9219063631145417, + "support": 20138.0 + }, + "cross-entropy": 0.1787492077720551, + "roc-auc": 0.9576967765860231, + "score": 0.9576967765860231 + }, + "test": { + "0": { + "precision": 0.9309967939376275, + "recall": 0.971856697345402, + "f1-score": 0.9509880540359495, + "support": 13147.0 + }, + "1": { + "precision": 0.8560871256320498, + "recall": 0.6991740787801779, + "f1-score": 0.7697149851372618, + "support": 3148.0 + }, + "accuracy": 0.9191776618594661, + "macro avg": { + "precision": 0.8935419597848386, + "recall": 0.8355153880627899, + "f1-score": 0.8603515195866056, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9165251378574827, + "recall": 0.9191776618594661, + "f1-score": 0.915968255269882, + "support": 16295.0 + }, + "cross-entropy": 0.18291214189215663, + "roc-auc": 0.9617315500639867, + "score": 0.9617315500639867 + } + }, + "time": "0:03:52.600341", + "diversity": { + "train": { + "mean_centered_corr": 0.0775546282529831, + "mean_pairwise_disagreement": 0.0356185432101606, + "member_std": 0.03550577908754349 + }, + "val": { + "mean_centered_corr": 0.08212897181510925, + "mean_pairwise_disagreement": 0.03171593825186856, + "member_std": 0.03616226091980934 + }, + "test": { + "mean_centered_corr": 0.10865671932697296, + "mean_pairwise_disagreement": 0.038570855892862445, + "member_std": 0.038954634219408035 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:01.289307" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/1/report.json new file mode 100644 index 000000000..bee7280b0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/1/report.json @@ -0,0 +1,657 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9036283493041992, + "min_features_per_member": 256.0, + "mean_features_per_member": 269.28125, + "max_features_per_member": 278.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.916107177734375, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8249481856886859, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9360805899283599, + "recall": 0.9794197976600667, + "f1-score": 0.9572599075643116, + "support": 181971.0 + }, + "1": { + "precision": 0.8896061785166843, + "recall": 0.7126260360339087, + "f1-score": 0.7913416280990652, + "support": 42349.0 + }, + "accuracy": 0.9290522467902995, + "macro avg": { + "precision": 0.9128433842225221, + "recall": 0.8460229168469877, + "f1-score": 0.8743007678316883, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.927306763034311, + "recall": 0.9290522467902995, + "f1-score": 0.9259364713255734, + "support": 224320.0 + }, + "cross-entropy": 0.16230668511159851, + "roc-auc": 0.968832958301438, + "score": 0.968832958301438 + }, + "val": { + "0": { + "precision": 0.9323972370060016, + "recall": 0.9836927304223164, + "f1-score": 0.9573583699096009, + "support": 16741.0 + }, + "1": { + "precision": 0.8897415185783522, + "recall": 0.6485133941713277, + "f1-score": 0.7502128384130768, + "support": 3397.0 + }, + "accuracy": 0.9271526467375112, + "macro avg": { + "precision": 0.9110693777921769, + "recall": 0.8161030622968221, + "f1-score": 0.8537856041613389, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9252018116659121, + "recall": 0.9271526467375112, + "f1-score": 0.9224158050822252, + "support": 20138.0 + }, + "cross-entropy": 0.17814126266590052, + "roc-auc": 0.9574705028701225, + "score": 0.9574705028701225 + }, + "test": { + "0": { + "precision": 0.9257767548906789, + "recall": 0.9790826804594204, + "f1-score": 0.951683856419356, + "support": 13147.0 + }, + "1": { + "precision": 0.8849853617733167, + "recall": 0.6721728081321474, + "f1-score": 0.764036829752663, + "support": 3148.0 + }, + "accuracy": 0.919791347038969, + "macro avg": { + "precision": 0.9053810583319978, + "recall": 0.8256277442957839, + "f1-score": 0.8578603430860094, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9178963433820286, + "recall": 0.919791347038969, + "f1-score": 0.9154326848976162, + "support": 16295.0 + }, + "cross-entropy": 0.18368392811429063, + "roc-auc": 0.9608438143835193, + "score": 0.9608438143835193 + } + }, + "time": "0:03:45.490338", + "diversity": { + "train": { + "mean_centered_corr": 0.09278353303670883, + "mean_pairwise_disagreement": 0.036516382126915464, + "member_std": 0.03687853366136551 + }, + "val": { + "mean_centered_corr": 0.09336636960506439, + "mean_pairwise_disagreement": 0.031406080303967146, + "member_std": 0.03779495880007744 + }, + "test": { + "mean_centered_corr": 0.11713645607233047, + "mean_pairwise_disagreement": 0.03774089617832504, + "member_std": 0.0392426960170269 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1 + ], + "head_selection_time": "0:00:01.237046" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/2/report.json new file mode 100644 index 000000000..ab5de1ddd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-best-head-evaluation/2/report.json @@ -0,0 +1,657 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8982802033424377, + "min_features_per_member": 254.0, + "mean_features_per_member": 267.6875, + "max_features_per_member": 283.0, + "min_members_per_feature": 22.0, + "mean_members_per_feature": 28.744966506958008, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8155679605741598, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2200, + "metrics": { + "train": { + "0": { + "precision": 0.9481424148606811, + "recall": 0.9693852317127455, + "f1-score": 0.9586461568560319, + "support": 181971.0 + }, + "1": { + "precision": 0.8544366638795987, + "recall": 0.772178799971664, + "f1-score": 0.8112278438620211, + "support": 42349.0 + }, + "accuracy": 0.9321549572039943, + "macro avg": { + "precision": 0.9012895393701399, + "recall": 0.8707820158422048, + "f1-score": 0.8849370003590264, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9304518618636328, + "recall": 0.9321549572039943, + "f1-score": 0.9308152985420904, + "support": 224320.0 + }, + "cross-entropy": 0.15461660323716772, + "roc-auc": 0.9724192435222043, + "score": 0.9724192435222043 + }, + "val": { + "0": { + "precision": 0.9400900380930394, + "recall": 0.9729406845469207, + "f1-score": 0.9562333049578772, + "support": 16741.0 + }, + "1": { + "precision": 0.8389046941678521, + "recall": 0.6944362672946718, + "f1-score": 0.7598647125140925, + "support": 3397.0 + }, + "accuracy": 0.9259608699970205, + "macro avg": { + "precision": 0.8894973661304457, + "recall": 0.8336884759207963, + "f1-score": 0.8580490087359849, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9230214804749114, + "recall": 0.9259608699970205, + "f1-score": 0.9231086595843775, + "support": 20138.0 + }, + "cross-entropy": 0.17667054083105546, + "roc-auc": 0.9571173941905297, + "score": 0.9571173941905297 + }, + "test": { + "0": { + "precision": 0.9367480994907373, + "recall": 0.9653913440328592, + "f1-score": 0.9508540605334133, + "support": 13147.0 + }, + "1": { + "precision": 0.8343044428259286, + "recall": 0.7277636594663278, + "f1-score": 0.7774007465218866, + "support": 3148.0 + }, + "accuracy": 0.9194845044492176, + "macro avg": { + "precision": 0.8855262711583329, + "recall": 0.8465775017495936, + "f1-score": 0.8641274035276499, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9169572046652806, + "recall": 0.9194845044492176, + "f1-score": 0.9173449453135124, + "support": 16295.0 + }, + "cross-entropy": 0.18060997930793665, + "roc-auc": 0.9613875873721536, + "score": 0.9613875873721536 + } + }, + "time": "0:03:57.910375", + "diversity": { + "train": { + "mean_centered_corr": 0.11927956342697144, + "mean_pairwise_disagreement": 0.04567602697471354, + "member_std": 0.04593726247549057 + }, + "val": { + "mean_centered_corr": 0.10152681171894073, + "mean_pairwise_disagreement": 0.042470762224521764, + "member_std": 0.04728136211633682 + }, + "test": { + "mean_centered_corr": 0.12282225489616394, + "mean_pairwise_disagreement": 0.05012644884142177, + "member_std": 0.0499299056828022 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 26 + ], + "head_selection_time": "0:00:01.531446" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml new file mode 100644 index 000000000..10f998b1d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml @@ -0,0 +1,58 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/0/report.json new file mode 100644 index 000000000..4fb8df0ec --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/0/report.json @@ -0,0 +1,653 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9398710962062461, + "recall": 0.9856790367695951, + "f1-score": 0.9622301917057169, + "support": 181971.0 + }, + "1": { + "precision": 0.9221624850657109, + "recall": 0.7290372854140593, + "f1-score": 0.8143058724234792, + "support": 42349.0 + }, + "accuracy": 0.9372280670470756, + "macro avg": { + "precision": 0.9310167906359785, + "recall": 0.8573581610918273, + "f1-score": 0.888268032064598, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.93652791693917, + "recall": 0.9372280670470756, + "f1-score": 0.9343038053055588, + "support": 224320.0 + }, + "cross-entropy": 0.14711870269948546, + "roc-auc": 0.9762437618004894, + "score": 0.9762437618004894 + }, + "val": { + "0": { + "precision": 0.9303347987825499, + "recall": 0.9859626067737889, + "f1-score": 0.9573412986109098, + "support": 16741.0 + }, + "1": { + "precision": 0.9019198664440734, + "recall": 0.6361495437150427, + "f1-score": 0.7460728465389262, + "support": 3397.0 + }, + "accuracy": 0.9269540172807628, + "macro avg": { + "precision": 0.9161273326133117, + "recall": 0.8110560752444158, + "f1-score": 0.8517070725749181, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9255415956266354, + "recall": 0.9269540172807628, + "f1-score": 0.9217032545306373, + "support": 20138.0 + }, + "cross-entropy": 0.175030748761162, + "roc-auc": 0.9589238560635404, + "score": 0.9589238560635404 + }, + "test": { + "0": { + "precision": 0.925214899713467, + "recall": 0.9824294515859131, + "f1-score": 0.9529641789943557, + "support": 13147.0 + }, + "1": { + "precision": 0.9010706638115632, + "recall": 0.6683608640406608, + "f1-score": 0.7674630676636878, + "support": 3148.0 + }, + "accuracy": 0.9217551396133783, + "macro avg": { + "precision": 0.9131427817625151, + "recall": 0.825395157813287, + "f1-score": 0.8602136233290217, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9205505207862382, + "recall": 0.9217551396133783, + "f1-score": 0.9171275727673571, + "support": 16295.0 + }, + "cross-entropy": 0.17966720795996355, + "roc-auc": 0.9624954901031625, + "score": 0.9624954901031625 + } + }, + "time": "0:03:52.600341", + "diversity": { + "train": { + "mean_centered_corr": 0.0775546282529831, + "mean_pairwise_disagreement": 0.0356185432101606, + "member_std": 0.03550577908754349 + }, + "val": { + "mean_centered_corr": 0.08212897181510925, + "mean_pairwise_disagreement": 0.03171593825186856, + "member_std": 0.03616226091980934 + }, + "test": { + "mean_centered_corr": 0.10865671932697296, + "mean_pairwise_disagreement": 0.038570855892862445, + "member_std": 0.038954634219408035 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml new file mode 100644 index 000000000..6083e1ea7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml @@ -0,0 +1,58 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 1 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/1/report.json new file mode 100644 index 000000000..12d2077f8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/1/report.json @@ -0,0 +1,653 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9036283493041992, + "min_features_per_member": 256.0, + "mean_features_per_member": 269.28125, + "max_features_per_member": 278.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.916107177734375, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8249481856886859, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.937763762727287, + "recall": 0.9854427353809123, + "f1-score": 0.961012234923391, + "support": 181971.0 + }, + "1": { + "precision": 0.9199625343686739, + "recall": 0.7189780160098231, + "f1-score": 0.8071468334968057, + "support": 42349.0 + }, + "accuracy": 0.9351373038516405, + "macro avg": { + "precision": 0.9288631485479805, + "recall": 0.8522103756953677, + "f1-score": 0.8840795342100984, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9344030984095316, + "recall": 0.9351373038516405, + "f1-score": 0.9319642414987546, + "support": 224320.0 + }, + "cross-entropy": 0.1516097613554574, + "roc-auc": 0.9745948569838376, + "score": 0.9745948569838376 + }, + "val": { + "0": { + "precision": 0.9278495227400337, + "recall": 0.9870975449495251, + "f1-score": 0.9565569737489509, + "support": 16741.0 + }, + "1": { + "precision": 0.9072164948453608, + "recall": 0.6217250515160435, + "f1-score": 0.7378165938864629, + "support": 3397.0 + }, + "accuracy": 0.9254642963551495, + "macro avg": { + "precision": 0.9175330087926973, + "recall": 0.8044112982327842, + "f1-score": 0.847186783817707, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.92436901843185, + "recall": 0.9254642963551495, + "f1-score": 0.9196585195631891, + "support": 20138.0 + }, + "cross-entropy": 0.17620349607555666, + "roc-auc": 0.9587836483021374, + "score": 0.9587836483021374 + }, + "test": { + "0": { + "precision": 0.924933744001146, + "recall": 0.9822012626454705, + "f1-score": 0.9527076877674487, + "support": 13147.0 + }, + "1": { + "precision": 0.8997429305912596, + "recall": 0.6670902160101652, + "f1-score": 0.7661437431594309, + "support": 3148.0 + }, + "accuracy": 0.9213255599877263, + "macro avg": { + "precision": 0.9123383372962028, + "recall": 0.8246457393278178, + "f1-score": 0.8594257154634398, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9200671787593956, + "recall": 0.9213255599877263, + "f1-score": 0.9166657548048196, + "support": 16295.0 + }, + "cross-entropy": 0.18052122203960097, + "roc-auc": 0.9621208050227469, + "score": 0.9621208050227469 + } + }, + "time": "0:03:45.490338", + "diversity": { + "train": { + "mean_centered_corr": 0.09278353303670883, + "mean_pairwise_disagreement": 0.036516382126915464, + "member_std": 0.03687853366136551 + }, + "val": { + "mean_centered_corr": 0.09336636960506439, + "mean_pairwise_disagreement": 0.031406080303967146, + "member_std": 0.03779495880007744 + }, + "test": { + "mean_centered_corr": 0.11713645607233047, + "mean_pairwise_disagreement": 0.03774089617832504, + "member_std": 0.0392426960170269 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml new file mode 100644 index 000000000..8f8578ce5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml @@ -0,0 +1,58 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 2 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/2/report.json new file mode 100644 index 000000000..708114571 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-evaluation/2/report.json @@ -0,0 +1,653 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8982802033424377, + "min_features_per_member": 254.0, + "mean_features_per_member": 267.6875, + "max_features_per_member": 283.0, + "min_members_per_feature": 22.0, + "mean_members_per_feature": 28.744966506958008, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8155679605741598, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2200, + "metrics": { + "train": { + "0": { + "precision": 0.9447407813455335, + "recall": 0.9849921141280753, + "f1-score": 0.9644466565329101, + "support": 181971.0 + }, + "1": { + "precision": 0.921060238177824, + "recall": 0.752438074098562, + "f1-score": 0.8282539476249269, + "support": 42349.0 + }, + "accuracy": 0.9410886233951498, + "macro avg": { + "precision": 0.9329005097616787, + "recall": 0.8687150941133186, + "f1-score": 0.8963503020789185, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.94027017095587, + "recall": 0.9410886233951498, + "f1-score": 0.9387350613584086, + "support": 224320.0 + }, + "cross-entropy": 0.14547932311545947, + "roc-auc": 0.978555896080379, + "score": 0.978555896080379 + }, + "val": { + "0": { + "precision": 0.9321899586800249, + "recall": 0.9837524640105131, + "f1-score": 0.9572773773541037, + "support": 16741.0 + }, + "1": { + "precision": 0.8899231080534197, + "recall": 0.6473358846040624, + "f1-score": 0.7494887525562373, + "support": 3397.0 + }, + "accuracy": 0.9270036746449498, + "macro avg": { + "precision": 0.9110565333667222, + "recall": 0.8155441743072878, + "f1-score": 0.8533830649551705, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9250601299195432, + "recall": 0.9270036746449498, + "f1-score": 0.9222263316475612, + "support": 20138.0 + }, + "cross-entropy": 0.17711932627215854, + "roc-auc": 0.9585690329227027, + "score": 0.9585690329227027 + }, + "test": { + "0": { + "precision": 0.9298067877559881, + "recall": 0.9773332319160265, + "f1-score": 0.9529778239264258, + "support": 13147.0 + }, + "1": { + "precision": 0.8796445880452343, + "recall": 0.6918678526048284, + "f1-score": 0.7745376955903271, + "support": 3148.0 + }, + "accuracy": 0.9221847192390303, + "macro avg": { + "precision": 0.9047256879006111, + "recall": 0.8346005422604275, + "f1-score": 0.8637577597583765, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9201160479775621, + "recall": 0.9221847192390303, + "f1-score": 0.9185053155494981, + "support": 16295.0 + }, + "cross-entropy": 0.181806240059393, + "roc-auc": 0.9620638665180715, + "score": 0.9620638665180715 + } + }, + "time": "0:03:57.910375", + "diversity": { + "train": { + "mean_centered_corr": 0.11927956342697144, + "mean_pairwise_disagreement": 0.04567602697471354, + "member_std": 0.04593726247549057 + }, + "val": { + "mean_centered_corr": 0.10152681171894073, + "mean_pairwise_disagreement": 0.042470762224521764, + "member_std": 0.04728136211633682 + }, + "test": { + "mean_centered_corr": 0.12282225489616394, + "mean_pairwise_disagreement": 0.05012644884142177, + "member_std": 0.0499299056828022 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..fb5104edb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,666 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9379184990964076, + "recall": 0.986816580663952, + "f1-score": 0.9617464096253906, + "support": 181971.0 + }, + "1": { + "precision": 0.9269977481589678, + "recall": 0.7193322156367329, + "f1-score": 0.8100676762707583, + "support": 42349.0 + }, + "accuracy": 0.9363186519258203, + "macro avg": { + "precision": 0.9324581236276877, + "recall": 0.8530743981503424, + "f1-score": 0.8859070429480744, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9358567886762504, + "recall": 0.9363186519258203, + "f1-score": 0.9331112336320092, + "support": 224320.0 + }, + "cross-entropy": 0.14950830177513505, + "roc-auc": 0.9756752367295682, + "score": 0.9756752367295682 + }, + "val": { + "0": { + "precision": 0.9291661978170361, + "recall": 0.9865002090675586, + "f1-score": 0.9569752281616688, + "support": 16741.0 + }, + "1": { + "precision": 0.9043993231810491, + "recall": 0.6293788637032676, + "f1-score": 0.7422322513452526, + "support": 3397.0 + }, + "accuracy": 0.9262588141821432, + "macro avg": { + "precision": 0.9167827604990426, + "recall": 0.8079395363854132, + "f1-score": 0.8496037397534607, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9249883711640194, + "recall": 0.9262588141821432, + "f1-score": 0.9207510801705393, + "support": 20138.0 + }, + "cross-entropy": 0.1746148873886249, + "roc-auc": 0.9597570226838343, + "score": 0.9597570226838343 + }, + "test": { + "0": { + "precision": 0.9218071860547847, + "recall": 0.9854719707918156, + "f1-score": 0.9525770163958532, + "support": 13147.0 + }, + "1": { + "precision": 0.9147321428571429, + "recall": 0.6508894536213469, + "f1-score": 0.7605790645879733, + "support": 3148.0 + }, + "accuracy": 0.920834611844124, + "macro avg": { + "precision": 0.9182696644559638, + "recall": 0.8181807122065812, + "f1-score": 0.8565780404919132, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9204403719408739, + "recall": 0.920834611844124, + "f1-score": 0.9154852979367427, + "support": 16295.0 + }, + "cross-entropy": 0.17973653945792817, + "roc-auc": 0.9633062567165207, + "score": 0.9633062567165207 + } + }, + "time": "0:03:52.600341", + "diversity": { + "train": { + "mean_centered_corr": 0.0775546282529831, + "mean_pairwise_disagreement": 0.0356185432101606, + "member_std": 0.03550577908754349 + }, + "val": { + "mean_centered_corr": 0.08212897181510925, + "mean_pairwise_disagreement": 0.03171593825186856, + "member_std": 0.03616226091980934 + }, + "test": { + "mean_centered_corr": 0.10865671932697296, + "mean_pairwise_disagreement": 0.038570855892862445, + "member_std": 0.038954634219408035 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 28, + 30, + 17, + 22, + 12, + 29, + 28, + 18, + 6 + ], + "head_selection_time": "0:00:13.698772" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..cae1077d0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1/report.json @@ -0,0 +1,671 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9036283493041992, + "min_features_per_member": 256.0, + "mean_features_per_member": 269.28125, + "max_features_per_member": 278.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.916107177734375, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8249481856886859, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9365119066228962, + "recall": 0.9876628693583044, + "f1-score": 0.9614075066665597, + "support": 181971.0 + }, + "1": { + "precision": 0.9307312557852515, + "recall": 0.7122954497154597, + "f1-score": 0.8069931379499459, + "support": 42349.0 + }, + "accuracy": 0.9356767118402283, + "macro avg": { + "precision": 0.9336215812040738, + "recall": 0.849979159536882, + "f1-score": 0.8842003223082529, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9354205871581875, + "recall": 0.9356767118402283, + "f1-score": 0.9322558746195738, + "support": 224320.0 + }, + "cross-entropy": 0.15071028513275977, + "roc-auc": 0.9752016380773885, + "score": 0.9752016380773885 + }, + "val": { + "0": { + "precision": 0.9268197088465846, + "recall": 0.9887700854190311, + "f1-score": 0.9567931562671599, + "support": 16741.0 + }, + "1": { + "precision": 0.9174714661984197, + "recall": 0.6152487488960848, + "f1-score": 0.7365638766519824, + "support": 3397.0 + }, + "accuracy": 0.9257622405402721, + "macro avg": { + "precision": 0.9221455875225022, + "recall": 0.802009417157558, + "f1-score": 0.8466785164595712, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9252427905689098, + "recall": 0.9257622405402721, + "f1-score": 0.9196435454392347, + "support": 20138.0 + }, + "cross-entropy": 0.1778182330217696, + "roc-auc": 0.9595148440428459, + "score": 0.9595148440428459 + }, + "test": { + "0": { + "precision": 0.9237536656891495, + "recall": 0.9823533886057656, + "f1-score": 0.9521527572987319, + "support": 13147.0 + }, + "1": { + "precision": 0.8997407087294728, + "recall": 0.6613722998729352, + "f1-score": 0.7623581105822044, + "support": 3148.0 + }, + "accuracy": 0.9203436637005217, + "macro avg": { + "precision": 0.9117471872093111, + "recall": 0.8218628442393504, + "f1-score": 0.8572554339404681, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9191146482906185, + "recall": 0.9203436637005217, + "f1-score": 0.9154866911518384, + "support": 16295.0 + }, + "cross-entropy": 0.1817569672472227, + "roc-auc": 0.9624332721317901, + "score": 0.9624332721317901 + } + }, + "time": "0:03:45.490338", + "diversity": { + "train": { + "mean_centered_corr": 0.09278353303670883, + "mean_pairwise_disagreement": 0.036516382126915464, + "member_std": 0.03687853366136551 + }, + "val": { + "mean_centered_corr": 0.09336636960506439, + "mean_pairwise_disagreement": 0.031406080303967146, + "member_std": 0.03779495880007744 + }, + "test": { + "mean_centered_corr": 0.11713645607233047, + "mean_pairwise_disagreement": 0.03774089617832504, + "member_std": 0.0392426960170269 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1, + 17, + 6, + 16, + 11, + 27, + 13, + 25, + 26, + 21, + 4, + 15, + 17, + 7, + 6 + ], + "head_selection_time": "0:00:16.127976" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..3ca863cda --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2/report.json @@ -0,0 +1,667 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8982802033424377, + "min_features_per_member": 254.0, + "mean_features_per_member": 267.6875, + "max_features_per_member": 283.0, + "min_members_per_feature": 22.0, + "mean_members_per_feature": 28.744966506958008, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8155679605741598, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2200, + "metrics": { + "train": { + "0": { + "precision": 0.9425560252475534, + "recall": 0.9839315055695688, + "f1-score": 0.9627994515096927, + "support": 181971.0 + }, + "1": { + "precision": 0.9149035243444603, + "recall": 0.7423315780774045, + "f1-score": 0.8196323816973016, + "support": 42349.0 + }, + "accuracy": 0.9383202567760343, + "macro avg": { + "precision": 0.9287297747960068, + "recall": 0.8631315418234866, + "f1-score": 0.8912159166034972, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9373355555580692, + "recall": 0.9383202567760343, + "f1-score": 0.935771178330819, + "support": 224320.0 + }, + "cross-entropy": 0.14788625943279124, + "roc-auc": 0.9766724082765327, + "score": 0.9766724082765327 + }, + "val": { + "0": { + "precision": 0.9320020365446625, + "recall": 0.984110865539693, + "f1-score": 0.9573478993549887, + "support": 16741.0 + }, + "1": { + "precision": 0.8919138561560341, + "recall": 0.6461583750367972, + "f1-score": 0.7494025264595425, + "support": 3397.0 + }, + "accuracy": 0.927102989373324, + "macro avg": { + "precision": 0.9119579463503483, + "recall": 0.815134620288245, + "f1-score": 0.8533752129072656, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9252397190960494, + "recall": 0.927102989373324, + "f1-score": 0.92227041242849, + "support": 20138.0 + }, + "cross-entropy": 0.17472041508549696, + "roc-auc": 0.9596270313530297, + "score": 0.9596270313530297 + }, + "test": { + "0": { + "precision": 0.9284322920424212, + "recall": 0.9788544915189777, + "f1-score": 0.9529768957345972, + "support": 13147.0 + }, + "1": { + "precision": 0.885784716516023, + "recall": 0.6848792884371029, + "f1-score": 0.7724829810103906, + "support": 3148.0 + }, + "accuracy": 0.9220619822031298, + "macro avg": { + "precision": 0.9071085042792221, + "recall": 0.8318668899780404, + "f1-score": 0.8627299383724939, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9201932881911109, + "recall": 0.9220619822031298, + "f1-score": 0.9181076202788253, + "support": 16295.0 + }, + "cross-entropy": 0.17945947055912823, + "roc-auc": 0.9628075343716237, + "score": 0.9628075343716237 + } + }, + "time": "0:03:57.910375", + "diversity": { + "train": { + "mean_centered_corr": 0.11927956342697144, + "mean_pairwise_disagreement": 0.04567602697471354, + "member_std": 0.04593726247549057 + }, + "val": { + "mean_centered_corr": 0.10152681171894073, + "mean_pairwise_disagreement": 0.042470762224521764, + "member_std": 0.04728136211633682 + }, + "test": { + "mean_centered_corr": 0.12282225489616394, + "mean_pairwise_disagreement": 0.05012644884142177, + "member_std": 0.0499299056828022 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 26, + 5, + 23, + 31, + 4, + 30, + 16, + 14, + 0, + 25, + 8 + ], + "head_selection_time": "0:00:22.293435" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f143c209a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/0/report.json @@ -0,0 +1,272 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9545321160743397, + "recall": 0.9652801820070231, + "f1-score": 0.9598760625261277, + "support": 181971.0 + }, + "1": { + "precision": 0.8432258064516129, + "recall": 0.8024274481097546, + "f1-score": 0.8223208992244311, + "support": 42349.0 + }, + "accuracy": 0.934535485021398, + "macro avg": { + "precision": 0.8988789612629764, + "recall": 0.8838538150583889, + "f1-score": 0.8910984808752793, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9335187828619073, + "recall": 0.934535485021398, + "f1-score": 0.9339072518509157, + "support": 224320.0 + }, + "cross-entropy": 0.1498442336366005, + "roc-auc": 0.9747257011404723, + "score": 0.9747257011404723 + }, + "val": { + "0": { + "precision": 0.9430700273589848, + "recall": 0.9677438623738128, + "f1-score": 0.955247641509434, + "support": 16741.0 + }, + "1": { + "precision": 0.8175059141601893, + "recall": 0.7120989108036503, + "f1-score": 0.7611705475141598, + "support": 3397.0 + }, + "accuracy": 0.9246201211639686, + "macro avg": { + "precision": 0.880287970759587, + "recall": 0.8399213865887316, + "f1-score": 0.8582090945117968, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9218891110546691, + "recall": 0.9246201211639686, + "f1-score": 0.9225095399947877, + "support": 20138.0 + }, + "cross-entropy": 0.18072654373317865, + "roc-auc": 0.9561512908829329, + "score": 0.9561512908829329 + }, + "test": { + "0": { + "precision": 0.938474098263322, + "recall": 0.9618163839659238, + "f1-score": 0.9500018782164457, + "support": 13147.0 + }, + "1": { + "precision": 0.8220489188231124, + "recall": 0.7366581956797967, + "f1-score": 0.7770145753057464, + "support": 3148.0 + }, + "accuracy": 0.9183185026081621, + "macro avg": { + "precision": 0.8802615085432172, + "recall": 0.8492372898228603, + "f1-score": 0.8635082267610961, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9159821396945721, + "recall": 0.9183185026081621, + "f1-score": 0.9165827907931329, + "support": 16295.0 + }, + "cross-entropy": 0.18568286621420937, + "roc-auc": 0.9606910602029306, + "score": 0.9606910602029306 + } + }, + "time": "0:05:02.257608", + "diversity": { + "train": { + "mean_centered_corr": 0.049995824694633484, + "mean_pairwise_disagreement": 0.03397909919872533, + "member_std": 0.031108465045690536 + }, + "val": { + "mean_centered_corr": 0.05969587340950966, + "mean_pairwise_disagreement": 0.03266793800197989, + "member_std": 0.03460051864385605 + }, + "test": { + "mean_centered_corr": 0.07376471161842346, + "mean_pairwise_disagreement": 0.04182474833958566, + "member_std": 0.03703681379556656 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 17 + ], + "head_selection_time": "0:00:00.953794" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..256963311 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/1/report.json @@ -0,0 +1,272 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.941301001780827, + "recall": 0.9847008589280709, + "f1-score": 0.9625119516990213, + "support": 181971.0 + }, + "1": { + "precision": 0.9180187873612298, + "recall": 0.7361448912607146, + "f1-score": 0.8170833988572627, + "support": 42349.0 + }, + "accuracy": 0.9377763908701855, + "macro avg": { + "precision": 0.9296598945710284, + "recall": 0.8604228750943927, + "f1-score": 0.889797675278142, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9369055912135325, + "recall": 0.9377763908701855, + "f1-score": 0.9350567368974181, + "support": 224320.0 + }, + "cross-entropy": 0.14229130006796945, + "roc-auc": 0.9765326990887259, + "score": 0.9765326990887259 + }, + "val": { + "0": { + "precision": 0.9301342054809969, + "recall": 0.9853055373036258, + "f1-score": 0.9569253081943437, + "support": 16741.0 + }, + "1": { + "precision": 0.8976705490848585, + "recall": 0.6352664115395937, + "f1-score": 0.7440096535080158, + "support": 3397.0 + }, + "accuracy": 0.9262588141821432, + "macro avg": { + "precision": 0.9139023772829278, + "recall": 0.8102859744216098, + "f1-score": 0.8504674808511798, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9246580389908946, + "recall": 0.9262588141821432, + "f1-score": 0.921009403984916, + "support": 20138.0 + }, + "cross-entropy": 0.18049319389718857, + "roc-auc": 0.9562486459053205, + "score": 0.9562486459053205 + }, + "test": { + "0": { + "precision": 0.9297238372093023, + "recall": 0.973073705027763, + "f1-score": 0.9509049689671832, + "support": 13147.0 + }, + "1": { + "precision": 0.8603550295857988, + "recall": 0.6928208386277002, + "f1-score": 0.7675523491113848, + "support": 3148.0 + }, + "accuracy": 0.918932187787665, + "macro avg": { + "precision": 0.8950394333975505, + "recall": 0.8329472718277315, + "f1-score": 0.859228659039284, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.916322609446259, + "recall": 0.918932187787665, + "f1-score": 0.9154834257142802, + "support": 16295.0 + }, + "cross-entropy": 0.18623170204474931, + "roc-auc": 0.9594156280332771, + "score": 0.9594156280332771 + } + }, + "time": "0:04:54.388848", + "diversity": { + "train": { + "mean_centered_corr": 0.046915579587221146, + "mean_pairwise_disagreement": 0.03045741646438268, + "member_std": 0.02836863324046135 + }, + "val": { + "mean_centered_corr": 0.05156576633453369, + "mean_pairwise_disagreement": 0.027064564985471218, + "member_std": 0.029862722381949425 + }, + "test": { + "mean_centered_corr": 0.06818301975727081, + "mean_pairwise_disagreement": 0.03394842075047758, + "member_std": 0.03134872391819954 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27 + ], + "head_selection_time": "0:00:01.014267" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..1019d433f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-best-head-evaluation/2/report.json @@ -0,0 +1,272 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9469624787748779, + "recall": 0.9776447895543795, + "f1-score": 0.9620590636981597, + "support": 181971.0 + }, + "1": { + "precision": 0.8884042465640688, + "recall": 0.7647169944980992, + "f1-score": 0.8219334534656481, + "support": 42349.0 + }, + "accuracy": 0.9374465049928673, + "macro avg": { + "precision": 0.9176833626694734, + "recall": 0.8711808920262394, + "f1-score": 0.8919962585819039, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9359073674344021, + "recall": 0.9374465049928673, + "f1-score": 0.9356049826187347, + "support": 224320.0 + }, + "cross-entropy": 0.14571912513044444, + "roc-auc": 0.975172867812646, + "score": 0.975172867812646 + }, + "val": { + "0": { + "precision": 0.9367074080449876, + "recall": 0.9750910937219999, + "f1-score": 0.9555139311636619, + "support": 16741.0 + }, + "1": { + "precision": 0.8461822205828108, + "recall": 0.6753017368266118, + "f1-score": 0.7511460379829732, + "support": 3397.0 + }, + "accuracy": 0.9245208064355944, + "macro avg": { + "precision": 0.8914448143138992, + "recall": 0.8251964152743059, + "f1-score": 0.8533299845733175, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9214370702850803, + "recall": 0.9245208064355944, + "f1-score": 0.9210399152169543, + "support": 20138.0 + }, + "cross-entropy": 0.18217191919121103, + "roc-auc": 0.9550014764588559, + "score": 0.9550014764588559 + }, + "test": { + "0": { + "precision": 0.9363965428085986, + "recall": 0.9641743363504982, + "f1-score": 0.9500824464098336, + "support": 13147.0 + }, + "1": { + "precision": 0.8292240754169689, + "recall": 0.7264930114358322, + "f1-score": 0.7744666440907552, + "support": 3148.0 + }, + "accuracy": 0.9182571340902117, + "macro avg": { + "precision": 0.8828103091127837, + "recall": 0.8453336738931652, + "f1-score": 0.8622745452502945, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9156920980495405, + "recall": 0.9182571340902117, + "f1-score": 0.9161555641944019, + "support": 16295.0 + }, + "cross-entropy": 0.188877181233535, + "roc-auc": 0.9593146174587831, + "score": 0.9593146174587831 + } + }, + "time": "0:05:03.267508", + "diversity": { + "train": { + "mean_centered_corr": 0.057024113833904266, + "mean_pairwise_disagreement": 0.032304333383185314, + "member_std": 0.030768897384405136 + }, + "val": { + "mean_centered_corr": 0.06899179518222809, + "mean_pairwise_disagreement": 0.0302923937732869, + "member_std": 0.03341741859912872 + }, + "test": { + "mean_centered_corr": 0.08463822305202484, + "mean_pairwise_disagreement": 0.03738159340387413, + "member_std": 0.035034921020269394 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:01.057452" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml new file mode 100644 index 000000000..016b99992 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0/report.json new file mode 100644 index 000000000..b0473609b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0/report.json @@ -0,0 +1,268 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9496102021429594, + "recall": 0.9833215182638992, + "f1-score": 0.9661718889206862, + "support": 181971.0 + }, + "1": { + "precision": 0.9154336983476832, + "recall": 0.7757916361661432, + "f1-score": 0.8398476443671873, + "support": 42349.0 + }, + "accuracy": 0.9441422967189729, + "macro avg": { + "precision": 0.9325219502453213, + "recall": 0.8795565772150212, + "f1-score": 0.9030097666439367, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9431580767897758, + "recall": 0.9441422967189729, + "f1-score": 0.9423233447311528, + "support": 224320.0 + }, + "cross-entropy": 0.13329761494760606, + "roc-auc": 0.9810524262916843, + "score": 0.9810524262916843 + }, + "val": { + "0": { + "precision": 0.936472602739726, + "recall": 0.9800489815423212, + "f1-score": 0.9577653892180613, + "support": 16741.0 + }, + "1": { + "precision": 0.8724216959511077, + "recall": 0.6723579629084486, + "f1-score": 0.7594347464671655, + "support": 3397.0 + }, + "accuracy": 0.9281457940212533, + "macro avg": { + "precision": 0.9044471493454169, + "recall": 0.8262034722253849, + "f1-score": 0.8586000678426133, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9256681072406231, + "recall": 0.9281457940212533, + "f1-score": 0.9243097732966793, + "support": 20138.0 + }, + "cross-entropy": 0.1734408298186842, + "roc-auc": 0.9583120395781356, + "score": 0.9583120395781356 + }, + "test": { + "0": { + "precision": 0.9324796274738067, + "recall": 0.9748231535711569, + "f1-score": 0.9531813617939088, + "support": 13147.0 + }, + "1": { + "precision": 0.8702469619756958, + "recall": 0.7052096569250318, + "f1-score": 0.779084049833304, + "support": 3148.0 + }, + "accuracy": 0.922737035900583, + "macro avg": { + "precision": 0.9013632947247513, + "recall": 0.8400164052480943, + "f1-score": 0.8661327058136064, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9204570174101029, + "recall": 0.922737035900583, + "f1-score": 0.9195478338373586, + "support": 16295.0 + }, + "cross-entropy": 0.17694458755574416, + "roc-auc": 0.9627483366901237, + "score": 0.9627483366901237 + } + }, + "time": "0:05:02.257608", + "diversity": { + "train": { + "mean_centered_corr": 0.049995824694633484, + "mean_pairwise_disagreement": 0.03397909919872533, + "member_std": 0.031108465045690536 + }, + "val": { + "mean_centered_corr": 0.05969587340950966, + "mean_pairwise_disagreement": 0.03266793800197989, + "member_std": 0.03460051864385605 + }, + "test": { + "mean_centered_corr": 0.07376471161842346, + "mean_pairwise_disagreement": 0.04182474833958566, + "member_std": 0.03703681379556656 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml new file mode 100644 index 000000000..d6bf43167 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1/report.json new file mode 100644 index 000000000..5c49bdfdd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1/report.json @@ -0,0 +1,268 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9423287951737644, + "recall": 0.9862835286941326, + "f1-score": 0.9638052788443466, + "support": 181971.0 + }, + "1": { + "precision": 0.9262868787100204, + "recall": 0.7406314198682378, + "f1-score": 0.823120325416612, + "support": 42349.0 + }, + "accuracy": 0.9399072753209701, + "macro avg": { + "precision": 0.9343078369418925, + "recall": 0.8634574742811851, + "f1-score": 0.8934628021304793, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9393002684248205, + "recall": 0.9399072753209701, + "f1-score": 0.9372456002926745, + "support": 224320.0 + }, + "cross-entropy": 0.13937913474355193, + "roc-auc": 0.9786839612084102, + "score": 0.9786839612084102 + }, + "val": { + "0": { + "precision": 0.930703684893629, + "recall": 0.9851860701272326, + "f1-score": 0.9571702164703151, + "support": 16741.0 + }, + "1": { + "precision": 0.8973934629706247, + "recall": 0.6385045628495731, + "f1-score": 0.7461300309597523, + "support": 3397.0 + }, + "accuracy": 0.9267057304598272, + "macro avg": { + "precision": 0.9140485739321269, + "recall": 0.8118453164884029, + "f1-score": 0.8516501237150337, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9250847145950668, + "recall": 0.9267057304598272, + "f1-score": 0.9215706777783208, + "support": 20138.0 + }, + "cross-entropy": 0.17362094113783183, + "roc-auc": 0.9584753793781823, + "score": 0.9584753793781823 + }, + "test": { + "0": { + "precision": 0.927138027763792, + "recall": 0.9804518141020765, + "f1-score": 0.9530499075785582, + "support": 13147.0 + }, + "1": { + "precision": 0.8925585284280937, + "recall": 0.6782083862770013, + "f1-score": 0.7707581227436823, + "support": 3148.0 + }, + "accuracy": 0.9220619822031298, + "macro avg": { + "precision": 0.9098482780959428, + "recall": 0.8293301001895389, + "f1-score": 0.8619040151611203, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9204576801781045, + "recall": 0.9220619822031298, + "f1-score": 0.9178333050219342, + "support": 16295.0 + }, + "cross-entropy": 0.17768448825468044, + "roc-auc": 0.9624917932683585, + "score": 0.9624917932683585 + } + }, + "time": "0:04:54.388848", + "diversity": { + "train": { + "mean_centered_corr": 0.046915579587221146, + "mean_pairwise_disagreement": 0.03045741646438268, + "member_std": 0.02836863324046135 + }, + "val": { + "mean_centered_corr": 0.05156576633453369, + "mean_pairwise_disagreement": 0.027064564985471218, + "member_std": 0.029862722381949425 + }, + "test": { + "mean_centered_corr": 0.06818301975727081, + "mean_pairwise_disagreement": 0.03394842075047758, + "member_std": 0.03134872391819954 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml new file mode 100644 index 000000000..856c5b853 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2/report.json new file mode 100644 index 000000000..3ab8d3d63 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2/report.json @@ -0,0 +1,268 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.947000732878843, + "recall": 0.9870309005281062, + "f1-score": 0.9666015488356824, + "support": 181971.0 + }, + "1": { + "precision": 0.9319040886401015, + "recall": 0.762639023353562, + "f1-score": 0.8388177544606914, + "support": 42349.0 + }, + "accuracy": 0.9446683309557775, + "macro avg": { + "precision": 0.9394524107594723, + "recall": 0.8748349619408341, + "f1-score": 0.902709651648187, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9441506625022985, + "recall": 0.9446683309557775, + "f1-score": 0.9424774586609922, + "support": 224320.0 + }, + "cross-entropy": 0.13143490246529854, + "roc-auc": 0.981694255249284, + "score": 0.981694255249284 + }, + "val": { + "0": { + "precision": 0.9317641733620007, + "recall": 0.9836927304223164, + "f1-score": 0.9570245532471306, + "support": 16741.0 + }, + "1": { + "precision": 0.8892045454545454, + "recall": 0.6449808654695319, + "f1-score": 0.7476539839617813, + "support": 3397.0 + }, + "accuracy": 0.9265567583672659, + "macro avg": { + "precision": 0.910484359408273, + "recall": 0.8143367979459242, + "f1-score": 0.852339268604456, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9245849571537563, + "recall": 0.9265567583672659, + "f1-score": 0.9217066555481371, + "support": 20138.0 + }, + "cross-entropy": 0.17442511117528403, + "roc-auc": 0.9582681142018287, + "score": 0.9582681142018287 + }, + "test": { + "0": { + "precision": 0.9284271284271284, + "recall": 0.9787784285388301, + "f1-score": 0.9529381271522198, + "support": 13147.0 + }, + "1": { + "precision": 0.8854209445585216, + "recall": 0.6848792884371029, + "f1-score": 0.7723446175891098, + "support": 3148.0 + }, + "accuracy": 0.9220006136851795, + "macro avg": { + "precision": 0.906924036492825, + "recall": 0.8318288584879665, + "f1-score": 0.8626413723706647, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9201188457135123, + "recall": 0.9220006136851795, + "f1-score": 0.9180496111592974, + "support": 16295.0 + }, + "cross-entropy": 0.17762575038093437, + "roc-auc": 0.9627356393915001, + "score": 0.9627356393915001 + } + }, + "time": "0:05:03.267508", + "diversity": { + "train": { + "mean_centered_corr": 0.057024113833904266, + "mean_pairwise_disagreement": 0.032304333383185314, + "member_std": 0.030768897384405136 + }, + "val": { + "mean_centered_corr": 0.06899179518222809, + "mean_pairwise_disagreement": 0.0302923937732869, + "member_std": 0.03341741859912872 + }, + "test": { + "mean_centered_corr": 0.08463822305202484, + "mean_pairwise_disagreement": 0.03738159340387413, + "member_std": 0.035034921020269394 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..55f982fc5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,290 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9495084683651196, + "recall": 0.9824752295695468, + "f1-score": 0.9657105817533626, + "support": 181971.0 + }, + "1": { + "precision": 0.9114928811301379, + "recall": 0.7755082764646155, + "f1-score": 0.8380199030364889, + "support": 42349.0 + }, + "accuracy": 0.9434022824536377, + "macro avg": { + "precision": 0.9305006747476288, + "recall": 0.8789917530170812, + "f1-score": 0.9018652423949257, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9423315688295711, + "recall": 0.9434022824536377, + "f1-score": 0.9416040707290184, + "support": 224320.0 + }, + "cross-entropy": 0.13410595787789584, + "roc-auc": 0.9804656245919036, + "score": 0.9804656245919036 + }, + "val": { + "0": { + "precision": 0.9362795477903392, + "recall": 0.9795113792485515, + "f1-score": 0.957407677711283, + "support": 16741.0 + }, + "1": { + "precision": 0.8692835365853658, + "recall": 0.6714748307329997, + "f1-score": 0.7576814482644079, + "support": 3397.0 + }, + "accuracy": 0.927549905651008, + "macro avg": { + "precision": 0.9027815421878524, + "recall": 0.8254931049907757, + "f1-score": 0.8575445629878454, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9249782542128591, + "recall": 0.927549905651008, + "f1-score": 0.9237166457601939, + "support": 20138.0 + }, + "cross-entropy": 0.17278545942960094, + "roc-auc": 0.958991133281215, + "score": 0.958991133281215 + }, + "test": { + "0": { + "precision": 0.9319371727748691, + "recall": 0.9748231535711569, + "f1-score": 0.9528978772445073, + "support": 13147.0 + }, + "1": { + "precision": 0.8698387731026347, + "recall": 0.7026683608640406, + "f1-score": 0.7773677736777368, + "support": 3148.0 + }, + "accuracy": 0.9222460877569807, + "macro avg": { + "precision": 0.9008879729387519, + "recall": 0.8387457572175987, + "f1-score": 0.865132825461122, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9199405012702239, + "recall": 0.9222460877569807, + "f1-score": 0.9189875510077357, + "support": 16295.0 + }, + "cross-entropy": 0.1769733127255982, + "roc-auc": 0.9627938464179217, + "score": 0.9627938464179217 + } + }, + "time": "0:05:02.257608", + "diversity": { + "train": { + "mean_centered_corr": 0.049995824694633484, + "mean_pairwise_disagreement": 0.03397909919872533, + "member_std": 0.031108465045690536 + }, + "val": { + "mean_centered_corr": 0.05969587340950966, + "mean_pairwise_disagreement": 0.03266793800197989, + "member_std": 0.03460051864385605 + }, + "test": { + "mean_centered_corr": 0.07376471161842346, + "mean_pairwise_disagreement": 0.04182474833958566, + "member_std": 0.03703681379556656 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 17, + 23, + 5, + 26, + 21, + 24, + 30, + 9, + 28, + 15, + 22, + 12, + 20, + 29, + 23, + 26, + 5, + 21, + 30 + ], + "head_selection_time": "0:00:17.134347" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..2050513dd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,278 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9430724174373912, + "recall": 0.9856625506261987, + "f1-score": 0.9638972484952708, + "support": 181971.0 + }, + "1": { + "precision": 0.9235592276815798, + "recall": 0.7443387092965595, + "f1-score": 0.8243200836820084, + "support": 42349.0 + }, + "accuracy": 0.9401034236804565, + "macro avg": { + "precision": 0.9333158225594855, + "recall": 0.865000629961379, + "f1-score": 0.8941086660886396, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9393885547725871, + "recall": 0.9401034236804565, + "f1-score": 0.9375467075150781, + "support": 224320.0 + }, + "cross-entropy": 0.13900322030342313, + "roc-auc": 0.978510242788649, + "score": 0.978510242788649 + }, + "val": { + "0": { + "precision": 0.9310831404865384, + "recall": 0.9853652708918225, + "f1-score": 0.9574554530152649, + "support": 16741.0 + }, + "1": { + "precision": 0.8988021478727798, + "recall": 0.6405652045922873, + "f1-score": 0.7480233757304916, + "support": 3397.0 + }, + "accuracy": 0.9272023041016982, + "macro avg": { + "precision": 0.9149426441796591, + "recall": 0.8129652377420549, + "f1-score": 0.8527394143728783, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9256377868313125, + "recall": 0.9272023041016982, + "f1-score": 0.9221271797738122, + "support": 20138.0 + }, + "cross-entropy": 0.17295465116447717, + "roc-auc": 0.9591595812965608, + "score": 0.9591595812965608 + }, + "test": { + "0": { + "precision": 0.9273710782545979, + "recall": 0.9780177987373545, + "f1-score": 0.952021323856064, + "support": 13147.0 + }, + "1": { + "precision": 0.8810699588477366, + "recall": 0.6801143583227446, + "f1-score": 0.767658659017569, + "support": 3148.0 + }, + "accuracy": 0.9204664007364222, + "macro avg": { + "precision": 0.9042205185511673, + "recall": 0.8290660785300495, + "f1-score": 0.8598399914368164, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9184262532228213, + "recall": 0.9204664007364222, + "f1-score": 0.916404651937587, + "support": 16295.0 + }, + "cross-entropy": 0.1781920882065017, + "roc-auc": 0.9622548696496048, + "score": 0.9622548696496048 + } + }, + "time": "0:04:54.388848", + "diversity": { + "train": { + "mean_centered_corr": 0.046915579587221146, + "mean_pairwise_disagreement": 0.03045741646438268, + "member_std": 0.02836863324046135 + }, + "val": { + "mean_centered_corr": 0.05156576633453369, + "mean_pairwise_disagreement": 0.027064564985471218, + "member_std": 0.029862722381949425 + }, + "test": { + "mean_centered_corr": 0.06818301975727081, + "mean_pairwise_disagreement": 0.03394842075047758, + "member_std": 0.03134872391819954 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27, + 14, + 18, + 11, + 24, + 5, + 26 + ], + "head_selection_time": "0:00:08.881424" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..8a5d00460 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,283 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.945536038382021, + "recall": 0.987717823169626, + "f1-score": 0.9661667472988228, + "support": 181971.0 + }, + "1": { + "precision": 0.9347083053372673, + "recall": 0.7555314175069069, + "f1-score": 0.8356228780360407, + "support": 42349.0 + }, + "accuracy": 0.9438837375178317, + "macro avg": { + "precision": 0.9401221718596442, + "recall": 0.8716246203382665, + "f1-score": 0.9008948126674318, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9434918886552366, + "recall": 0.9438837375178317, + "f1-score": 0.9415215871730669, + "support": 224320.0 + }, + "cross-entropy": 0.13196222295418844, + "roc-auc": 0.9811511759509819, + "score": 0.9811511759509819 + }, + "val": { + "0": { + "precision": 0.9318708656075083, + "recall": 0.9845290006570695, + "f1-score": 0.9574764726385501, + "support": 16741.0 + }, + "1": { + "precision": 0.8943288453692371, + "recall": 0.6452752428613483, + "f1-score": 0.7496580027359782, + "support": 3397.0 + }, + "accuracy": 0.9273016188300724, + "macro avg": { + "precision": 0.9130998554883727, + "recall": 0.814902121759209, + "f1-score": 0.8535672376872641, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9255380498984306, + "recall": 0.9273016188300724, + "f1-score": 0.9224203924787012, + "support": 20138.0 + }, + "cross-entropy": 0.17394704368189032, + "roc-auc": 0.9590920051471822, + "score": 0.9590920051471822 + }, + "test": { + "0": { + "precision": 0.9284375900835976, + "recall": 0.9799193732410436, + "f1-score": 0.9534840691262998, + "support": 13147.0 + }, + "1": { + "precision": 0.8908639933856966, + "recall": 0.684561626429479, + "f1-score": 0.7742051374169211, + "support": 3148.0 + }, + "accuracy": 0.9228597729364836, + "macro avg": { + "precision": 0.909650791734647, + "recall": 0.8322404998352613, + "f1-score": 0.8638446032716105, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9211788185337362, + "recall": 0.9228597729364836, + "f1-score": 0.9188495139240216, + "support": 16295.0 + }, + "cross-entropy": 0.1774981138265784, + "roc-auc": 0.9631468941416912, + "score": 0.9631468941416912 + } + }, + "time": "0:05:03.267508", + "diversity": { + "train": { + "mean_centered_corr": 0.057024113833904266, + "mean_pairwise_disagreement": 0.032304333383185314, + "member_std": 0.030768897384405136 + }, + "val": { + "mean_centered_corr": 0.06899179518222809, + "mean_pairwise_disagreement": 0.0302923937732869, + "member_std": 0.03341741859912872 + }, + "test": { + "mean_centered_corr": 0.08463822305202484, + "mean_pairwise_disagreement": 0.03738159340387413, + "member_std": 0.035034921020269394 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 1, + 5, + 10, + 3, + 4, + 27, + 15, + 18, + 1, + 6, + 25 + ], + "head_selection_time": "0:00:12.525387" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a5901fec9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/0/report.json @@ -0,0 +1,298 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9506081938743591, + "min_features_per_member": 275.0, + "mean_features_per_member": 283.28125, + "max_features_per_member": 292.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.419462203979492, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9056525652610176, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9539299644574761, + "recall": 0.9690170411768908, + "f1-score": 0.9614143176489831, + "support": 181971.0 + }, + "1": { + "precision": 0.8571609536115122, + "recall": 0.7989090651491181, + "f1-score": 0.8270105108775361, + "support": 42349.0 + }, + "accuracy": 0.9369026390870185, + "macro avg": { + "precision": 0.9055454590344942, + "recall": 0.8839630531630045, + "f1-score": 0.8942124142632596, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9356611037214039, + "recall": 0.9369026390870185, + "f1-score": 0.9360404463358412, + "support": 224320.0 + }, + "cross-entropy": 0.14460702857706767, + "roc-auc": 0.9762667563935794, + "score": 0.9762667563935794 + }, + "val": { + "0": { + "precision": 0.9437383395522388, + "recall": 0.9669075921390597, + "f1-score": 0.9551824860590682, + "support": 16741.0 + }, + "1": { + "precision": 0.8144675150703282, + "recall": 0.7159258168972623, + "f1-score": 0.7620241265862447, + "support": 3397.0 + }, + "accuracy": 0.9245704637997815, + "macro avg": { + "precision": 0.8791029273112836, + "recall": 0.8414167045181611, + "f1-score": 0.8586033063226565, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9219321527032445, + "recall": 0.9245704637997815, + "f1-score": 0.9225993622568445, + "support": 20138.0 + }, + "cross-entropy": 0.1811185402192146, + "roc-auc": 0.9563975490624737, + "score": 0.9563975490624737 + }, + "test": { + "0": { + "precision": 0.9422366337380288, + "recall": 0.9504069369437894, + "f1-score": 0.9463041502574977, + "support": 13147.0 + }, + "1": { + "precision": 0.7851021753460777, + "recall": 0.7566709021601017, + "f1-score": 0.7706243934001941, + "support": 3148.0 + }, + "accuracy": 0.9129794415464867, + "macro avg": { + "precision": 0.8636694045420532, + "recall": 0.8535389195519456, + "f1-score": 0.8584642718288459, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.911880127139817, + "recall": 0.9129794415464867, + "f1-score": 0.9123649127866912, + "support": 16295.0 + }, + "cross-entropy": 0.19078130335337062, + "roc-auc": 0.9593891219693567, + "score": 0.9593891219693567 + } + }, + "time": "0:03:07.006918", + "diversity": { + "train": { + "mean_centered_corr": 0.08914346992969513, + "mean_pairwise_disagreement": 0.03838632562640928, + "member_std": 0.037065718322992325 + }, + "val": { + "mean_centered_corr": 0.09447728097438812, + "mean_pairwise_disagreement": 0.0372919796949436, + "member_std": 0.04158519208431244 + }, + "test": { + "mean_centered_corr": 0.11072365194559097, + "mean_pairwise_disagreement": 0.045909714042502645, + "member_std": 0.04358496516942978 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.952109" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..a9dfa7308 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/1/report.json @@ -0,0 +1,298 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9529152512550354, + "min_features_per_member": 274.0, + "mean_features_per_member": 283.96875, + "max_features_per_member": 290.0, + "min_members_per_feature": 25.0, + "mean_members_per_feature": 30.493288040161133, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9106552463862567, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9437590003006694, + "recall": 0.9832061152601239, + "f1-score": 0.9630787946644848, + "support": 181971.0 + }, + "1": { + "precision": 0.9120398353625191, + "recall": 0.7482349051925665, + "f1-score": 0.8220567633476885, + "support": 42349.0 + }, + "accuracy": 0.9388462910128388, + "macro avg": { + "precision": 0.9278994178315942, + "recall": 0.8657205102263452, + "f1-score": 0.8925677790060866, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9377707918664427, + "recall": 0.9388462910128388, + "f1-score": 0.9364554797383303, + "support": 224320.0 + }, + "cross-entropy": 0.14120537728706906, + "roc-auc": 0.9768407671782054, + "score": 0.9768407671782054 + }, + "val": { + "0": { + "precision": 0.9325459913695208, + "recall": 0.9810644525416642, + "f1-score": 0.9561901435100283, + "support": 16741.0 + }, + "1": { + "precision": 0.8745051464766429, + "recall": 0.6502796585222255, + "f1-score": 0.7459057909842985, + "support": 3397.0 + }, + "accuracy": 0.9252656668984011, + "macro avg": { + "precision": 0.9035255689230819, + "recall": 0.8156720555319448, + "f1-score": 0.8510479672471634, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9227553095688898, + "recall": 0.9252656668984011, + "f1-score": 0.9207181033109071, + "support": 20138.0 + }, + "cross-entropy": 0.18368600862444018, + "roc-auc": 0.9550071913296724, + "score": 0.9550071913296724 + }, + "test": { + "0": { + "precision": 0.9260352898811667, + "recall": 0.9780177987373545, + "f1-score": 0.951316957679787, + "support": 13147.0 + }, + "1": { + "precision": 0.8800829875518672, + "recall": 0.6737611181702668, + "f1-score": 0.7632241813602015, + "support": 3148.0 + }, + "accuracy": 0.9192390303774164, + "macro avg": { + "precision": 0.903059138716517, + "recall": 0.8258894584538107, + "f1-score": 0.8572705695199943, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9171578521559359, + "recall": 0.9192390303774164, + "f1-score": 0.9149796726319774, + "support": 16295.0 + }, + "cross-entropy": 0.18586409414781727, + "roc-auc": 0.9596843251014889, + "score": 0.9596843251014889 + } + }, + "time": "0:03:09.746628", + "diversity": { + "train": { + "mean_centered_corr": 0.08831919729709625, + "mean_pairwise_disagreement": 0.041946592713174725, + "member_std": 0.03971056640148163 + }, + "val": { + "mean_centered_corr": 0.10093759745359421, + "mean_pairwise_disagreement": 0.040990251939039976, + "member_std": 0.044565752148628235 + }, + "test": { + "mean_centered_corr": 0.11717803031206131, + "mean_pairwise_disagreement": 0.04949915370834117, + "member_std": 0.04652492329478264 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 11 + ], + "head_selection_time": "0:00:00.926202" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..1e68d7660 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-best-head-evaluation/2/report.json @@ -0,0 +1,298 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9485108852386475, + "min_features_per_member": 274.0, + "mean_features_per_member": 282.65625, + "max_features_per_member": 289.0, + "min_members_per_feature": 25.0, + "mean_members_per_feature": 30.35234832763672, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9021152843018349, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9364448961509395, + "recall": 0.9873606233960356, + "f1-score": 0.9612289881124343, + "support": 181971.0 + }, + "1": { + "precision": 0.9291326452010477, + "recall": 0.7120593166308532, + "f1-score": 0.8062403080049195, + "support": 42349.0 + }, + "accuracy": 0.9353869472182597, + "macro avg": { + "precision": 0.9327887706759936, + "recall": 0.8497099700134444, + "f1-score": 0.883734648058677, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9350644284464237, + "recall": 0.9353869472182597, + "f1-score": 0.931968932772415, + "support": 224320.0 + }, + "cross-entropy": 0.14661004133947955, + "roc-auc": 0.9751240053891049, + "score": 0.9751240053891049 + }, + "val": { + "0": { + "precision": 0.9273278670111198, + "recall": 0.9863210083029688, + "f1-score": 0.9559151301126002, + "support": 16741.0 + }, + "1": { + "precision": 0.9018010291595198, + "recall": 0.6190756549896967, + "f1-score": 0.7341595391865945, + "support": 3397.0 + }, + "accuracy": 0.924371834343033, + "macro avg": { + "precision": 0.9145644480853198, + "recall": 0.8026983316463328, + "f1-score": 0.8450373346495974, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9230218451528475, + "recall": 0.924371834343033, + "f1-score": 0.9185080518339408, + "support": 20138.0 + }, + "cross-entropy": 0.18460242926495607, + "roc-auc": 0.9545391258255768, + "score": 0.9545391258255768 + }, + "test": { + "0": { + "precision": 0.9253505933117584, + "recall": 0.9787023655586826, + "f1-score": 0.9512790181871951, + "support": 13147.0 + }, + "1": { + "precision": 0.8828451882845189, + "recall": 0.670266836086404, + "f1-score": 0.7620079451065367, + "support": 3148.0 + }, + "accuracy": 0.9191162933415158, + "macro avg": { + "precision": 0.9040978907981386, + "recall": 0.8244846008225433, + "f1-score": 0.8566434816468659, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9171390551082756, + "recall": 0.9191162933415158, + "f1-score": 0.9147141002333495, + "support": 16295.0 + }, + "cross-entropy": 0.18885611976418637, + "roc-auc": 0.9588200872762291, + "score": 0.9588200872762291 + } + }, + "time": "0:03:06.503534", + "diversity": { + "train": { + "mean_centered_corr": 0.08432834595441818, + "mean_pairwise_disagreement": 0.03995795716660531, + "member_std": 0.04036388918757439 + }, + "val": { + "mean_centered_corr": 0.0944766253232956, + "mean_pairwise_disagreement": 0.03774069805439243, + "member_std": 0.043501291424036026 + }, + "test": { + "mean_centered_corr": 0.10949686914682388, + "mean_pairwise_disagreement": 0.04558554969365232, + "member_std": 0.04584404453635216 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:01.499174" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml new file mode 100644 index 000000000..49f80d6e4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..24e827ce9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0/report.json @@ -0,0 +1,294 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9506081938743591, + "min_features_per_member": 275.0, + "mean_features_per_member": 283.28125, + "max_features_per_member": 292.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.419462203979492, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9056525652610176, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9496583046818625, + "recall": 0.9835852965582428, + "f1-score": 0.9663241038432363, + "support": 181971.0 + }, + "1": { + "precision": 0.9166759651863423, + "recall": 0.7759569293253678, + "f1-score": 0.8404670255892169, + "support": 42349.0 + }, + "accuracy": 0.9443874821683309, + "macro avg": { + "precision": 0.9331671349341024, + "recall": 0.8797711129418053, + "f1-score": 0.9033955647162266, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9434316236222344, + "recall": 0.9443874821683309, + "f1-score": 0.9425637552029926, + "support": 224320.0 + }, + "cross-entropy": 0.1349038769118999, + "roc-auc": 0.9810848735528083, + "score": 0.9810848735528083 + }, + "val": { + "0": { + "precision": 0.9357175450604609, + "recall": 0.979929514365928, + "f1-score": 0.9573133370291483, + "support": 16741.0 + }, + "1": { + "precision": 0.8710667689946278, + "recall": 0.6682366794230203, + "f1-score": 0.756288522405464, + "support": 3397.0 + }, + "accuracy": 0.9273512761942596, + "macro avg": { + "precision": 0.9033921570275443, + "recall": 0.8240830968944741, + "f1-score": 0.8568009297173061, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.924811859972784, + "recall": 0.9273512761942596, + "f1-score": 0.9234032518530308, + "support": 20138.0 + }, + "cross-entropy": 0.17405502211594576, + "roc-auc": 0.9584246137411132, + "score": 0.9584246137411132 + }, + "test": { + "0": { + "precision": 0.9328956965718453, + "recall": 0.9728455160873203, + "f1-score": 0.9524518747440146, + "support": 13147.0 + }, + "1": { + "precision": 0.8618955512572534, + "recall": 0.7077509529860229, + "f1-score": 0.7772544915402059, + "support": 3148.0 + }, + "accuracy": 0.9216324025774778, + "macro avg": { + "precision": 0.8973956239145493, + "recall": 0.8402982345366716, + "f1-score": 0.8648531831421102, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9191793137887625, + "recall": 0.9216324025774778, + "f1-score": 0.9186058261201674, + "support": 16295.0 + }, + "cross-entropy": 0.1782989047843598, + "roc-auc": 0.962318960200698, + "score": 0.962318960200698 + } + }, + "time": "0:03:07.006918", + "diversity": { + "train": { + "mean_centered_corr": 0.08914346992969513, + "mean_pairwise_disagreement": 0.03838632562640928, + "member_std": 0.037065718322992325 + }, + "val": { + "mean_centered_corr": 0.09447728097438812, + "mean_pairwise_disagreement": 0.0372919796949436, + "member_std": 0.04158519208431244 + }, + "test": { + "mean_centered_corr": 0.11072365194559097, + "mean_pairwise_disagreement": 0.045909714042502645, + "member_std": 0.04358496516942978 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml new file mode 100644 index 000000000..7df284f20 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml @@ -0,0 +1,47 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 1 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1/report.json new file mode 100644 index 000000000..f795b8ee6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1/report.json @@ -0,0 +1,294 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9529152512550354, + "min_features_per_member": 274.0, + "mean_features_per_member": 283.96875, + "max_features_per_member": 290.0, + "min_members_per_feature": 25.0, + "mean_members_per_feature": 30.493288040161133, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9106552463862567, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9512478050337891, + "recall": 0.9823982942336966, + "f1-score": 0.9665721368690542, + "support": 181971.0 + }, + "1": { + "precision": 0.911981313547678, + "recall": 0.7836548678835391, + "f1-score": 0.8429621915442157, + "support": 42349.0 + }, + "accuracy": 0.9448778530670471, + "macro avg": { + "precision": 0.9316145592907336, + "recall": 0.8830265810586179, + "f1-score": 0.9047671642066349, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9438347493635622, + "recall": 0.9448778530670471, + "f1-score": 0.9432360207199744, + "support": 224320.0 + }, + "cross-entropy": 0.13516445510887865, + "roc-auc": 0.9814280207794918, + "score": 0.9814280207794918 + }, + "val": { + "0": { + "precision": 0.9362831858407079, + "recall": 0.9795711128367481, + "f1-score": 0.9574381130312938, + "support": 16741.0 + }, + "1": { + "precision": 0.8696149447197865, + "recall": 0.6714748307329997, + "f1-score": 0.7578073089700996, + "support": 3397.0 + }, + "accuracy": 0.9275995630151952, + "macro avg": { + "precision": 0.9029490652802472, + "recall": 0.825522971784874, + "f1-score": 0.8576227110006966, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.925037182509306, + "recall": 0.9275995630151952, + "f1-score": 0.9237631780131254, + "support": 20138.0 + }, + "cross-entropy": 0.17412061097796955, + "roc-auc": 0.9585282902898349, + "score": 0.9585282902898349 + }, + "test": { + "0": { + "precision": 0.9320947176684882, + "recall": 0.973073705027763, + "f1-score": 0.9521434950878238, + "support": 13147.0 + }, + "1": { + "precision": 0.8622568093385214, + "recall": 0.7039390088945362, + "f1-score": 0.7750961874781392, + "support": 3148.0 + }, + "accuracy": 0.9210800859159252, + "macro avg": { + "precision": 0.8971757635035048, + "recall": 0.8385063569611496, + "f1-score": 0.8636198412829815, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9186028652338312, + "recall": 0.9210800859159252, + "f1-score": 0.9179400630930225, + "support": 16295.0 + }, + "cross-entropy": 0.17837645706043, + "roc-auc": 0.962430602195543, + "score": 0.962430602195543 + } + }, + "time": "0:03:09.746628", + "diversity": { + "train": { + "mean_centered_corr": 0.08831919729709625, + "mean_pairwise_disagreement": 0.041946592713174725, + "member_std": 0.03971056640148163 + }, + "val": { + "mean_centered_corr": 0.10093759745359421, + "mean_pairwise_disagreement": 0.040990251939039976, + "member_std": 0.044565752148628235 + }, + "test": { + "mean_centered_corr": 0.11717803031206131, + "mean_pairwise_disagreement": 0.04949915370834117, + "member_std": 0.04652492329478264 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml new file mode 100644 index 000000000..22936b0f3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml @@ -0,0 +1,47 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 2 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2/report.json new file mode 100644 index 000000000..f496b64bd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2/report.json @@ -0,0 +1,294 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9485108852386475, + "min_features_per_member": 274.0, + "mean_features_per_member": 282.65625, + "max_features_per_member": 289.0, + "min_members_per_feature": 25.0, + "mean_members_per_feature": 30.35234832763672, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9021152843018349, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9471737091181214, + "recall": 0.987684850882833, + "f1-score": 0.9670051785594189, + "support": 181971.0 + }, + "1": { + "precision": 0.9351675056413817, + "recall": 0.7633001959904602, + "f1-score": 0.8405382565169343, + "support": 42349.0 + }, + "accuracy": 0.9453236447931527, + "macro avg": { + "precision": 0.9411706073797516, + "recall": 0.8754925234366466, + "f1-score": 0.9037717175381765, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9449070779169959, + "recall": 0.9453236447931527, + "f1-score": 0.9431296985238573, + "support": 224320.0 + }, + "cross-entropy": 0.13396425577185272, + "roc-auc": 0.9820352766151116, + "score": 0.9820352766151116 + }, + "val": { + "0": { + "precision": 0.9312351678155724, + "recall": 0.9844692670688728, + "f1-score": 0.9571125758587647, + "support": 16741.0 + }, + "1": { + "precision": 0.8934426229508197, + "recall": 0.6417427141595525, + "f1-score": 0.7469590543087202, + "support": 3397.0 + }, + "accuracy": 0.9266560730956401, + "macro avg": { + "precision": 0.912338895383196, + "recall": 0.8131059906142126, + "f1-score": 0.8520358150837425, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9248600920927813, + "recall": 0.9266560730956401, + "f1-score": 0.9216626050222616, + "support": 20138.0 + }, + "cross-entropy": 0.17636787506935708, + "roc-auc": 0.9583243485306635, + "score": 0.9583243485306635 + }, + "test": { + "0": { + "precision": 0.9282006920415224, + "recall": 0.9793869323800106, + "f1-score": 0.9531070728006218, + "support": 13147.0 + }, + "1": { + "precision": 0.8881551795295088, + "recall": 0.6836086404066074, + "f1-score": 0.7725722491473703, + "support": 3148.0 + }, + "accuracy": 0.9222460877569807, + "macro avg": { + "precision": 0.9081779357855156, + "recall": 0.8314977863933091, + "f1-score": 0.8628396609739961, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9204643757857496, + "recall": 0.9222460877569807, + "f1-score": 0.9182298942267995, + "support": 16295.0 + }, + "cross-entropy": 0.1797688037922231, + "roc-auc": 0.9625461826483815, + "score": 0.9625461826483815 + } + }, + "time": "0:03:06.503534", + "diversity": { + "train": { + "mean_centered_corr": 0.08432834595441818, + "mean_pairwise_disagreement": 0.03995795716660531, + "member_std": 0.04036388918757439 + }, + "val": { + "mean_centered_corr": 0.0944766253232956, + "mean_pairwise_disagreement": 0.03774069805439243, + "member_std": 0.043501291424036026 + }, + "test": { + "mean_centered_corr": 0.10949686914682388, + "mean_pairwise_disagreement": 0.04558554969365232, + "member_std": 0.04584404453635216 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..633179f9a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,304 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9506081938743591, + "min_features_per_member": 275.0, + "mean_features_per_member": 283.28125, + "max_features_per_member": 292.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.419462203979492, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9056525652610176, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9503279377423628, + "recall": 0.9817773161657627, + "f1-score": 0.9657966721086376, + "support": 181971.0 + }, + "1": { + "precision": 0.9087180334186693, + "recall": 0.779498925594465, + "f1-score": 0.8391631501347298, + "support": 42349.0 + }, + "accuracy": 0.943589514978602, + "macro avg": { + "precision": 0.929522985580516, + "recall": 0.8806381208801138, + "f1-score": 0.9024799111216837, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9424724730570735, + "recall": 0.943589514978602, + "f1-score": 0.941889739948897, + "support": 224320.0 + }, + "cross-entropy": 0.13479785159389124, + "roc-auc": 0.9805680231536485, + "score": 0.9805680231536485 + }, + "val": { + "0": { + "precision": 0.9373999542648068, + "recall": 0.9794516456603548, + "f1-score": 0.9579645371425233, + "support": 16741.0 + }, + "1": { + "precision": 0.8699924414210128, + "recall": 0.6776567559611422, + "f1-score": 0.7618732417673341, + "support": 3397.0 + }, + "accuracy": 0.9285430529347503, + "macro avg": { + "precision": 0.9036961978429099, + "recall": 0.8285542008107485, + "f1-score": 0.8599188894549287, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9260292460946623, + "recall": 0.9285430529347503, + "f1-score": 0.9248866679206782, + "support": 20138.0 + }, + "cross-entropy": 0.17265539939712965, + "roc-auc": 0.9592957095194116, + "score": 0.9592957095194116 + }, + "test": { + "0": { + "precision": 0.9348464815630968, + "recall": 0.9680535483380239, + "f1-score": 0.9511602705429543, + "support": 13147.0 + }, + "1": { + "precision": 0.8433420365535248, + "recall": 0.7182337992376112, + "f1-score": 0.7757762909589981, + "support": 3148.0 + }, + "accuracy": 0.919791347038969, + "macro avg": { + "precision": 0.8890942590583109, + "recall": 0.8431436737878175, + "f1-score": 0.8634682807509761, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9171689121927296, + "recall": 0.919791347038969, + "f1-score": 0.9172781737199843, + "support": 16295.0 + }, + "cross-entropy": 0.1790101347961027, + "roc-auc": 0.9620218603265258, + "score": 0.9620218603265258 + } + }, + "time": "0:03:07.006918", + "diversity": { + "train": { + "mean_centered_corr": 0.08914346992969513, + "mean_pairwise_disagreement": 0.03838632562640928, + "member_std": 0.037065718322992325 + }, + "val": { + "mean_centered_corr": 0.09447728097438812, + "mean_pairwise_disagreement": 0.0372919796949436, + "member_std": 0.04158519208431244 + }, + "test": { + "mean_centered_corr": 0.11072365194559097, + "mean_pairwise_disagreement": 0.045909714042502645, + "member_std": 0.04358496516942978 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 5, + 28, + 20, + 9, + 6, + 23 + ], + "head_selection_time": "0:00:08.820400" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..b8a656ee7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,310 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9529152512550354, + "min_features_per_member": 274.0, + "mean_features_per_member": 283.96875, + "max_features_per_member": 290.0, + "min_members_per_feature": 25.0, + "mean_members_per_feature": 30.493288040161133, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9106552463862567, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9512209410699545, + "recall": 0.9815080424902869, + "f1-score": 0.9661271826384231, + "support": 181971.0 + }, + "1": { + "precision": 0.9079469292846396, + "recall": 0.7837257078089211, + "f1-score": 0.8412754739937138, + "support": 42349.0 + }, + "accuracy": 0.9441690442225392, + "macro avg": { + "precision": 0.929583935177297, + "recall": 0.882616875149604, + "f1-score": 0.9037013283160684, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9430513123025851, + "recall": 0.9441690442225392, + "f1-score": 0.9425566360558857, + "support": 224320.0 + }, + "cross-entropy": 0.13677446177980773, + "roc-auc": 0.9808759836686648, + "score": 0.9808759836686648 + }, + "val": { + "0": { + "precision": 0.9363148275074252, + "recall": 0.9792127113075683, + "f1-score": 0.9572834243335572, + "support": 16741.0 + }, + "1": { + "precision": 0.867680608365019, + "recall": 0.671769208124816, + "f1-score": 0.7572590011614402, + "support": 3397.0 + }, + "accuracy": 0.9273512761942596, + "macro avg": { + "precision": 0.9019977179362222, + "recall": 0.8254909597161921, + "f1-score": 0.8572712127474986, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9247371910774543, + "recall": 0.9273512761942596, + "f1-score": 0.9235420912560082, + "support": 20138.0 + }, + "cross-entropy": 0.1738401509177298, + "roc-auc": 0.9591725848256957, + "score": 0.9591725848256957 + }, + "test": { + "0": { + "precision": 0.9327056419239471, + "recall": 0.9720088233056972, + "f1-score": 0.9519517282479142, + "support": 13147.0 + }, + "1": { + "precision": 0.8581341557440246, + "recall": 0.7071156289707751, + "f1-score": 0.7753396029258098, + "support": 3148.0 + }, + "accuracy": 0.920834611844124, + "macro avg": { + "precision": 0.8954198988339859, + "recall": 0.8395622261382361, + "f1-score": 0.8636456655868621, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9182993186042542, + "recall": 0.920834611844124, + "f1-score": 0.9178323682900138, + "support": 16295.0 + }, + "cross-entropy": 0.17951798610400288, + "roc-auc": 0.9622278199335073, + "score": 0.9622278199335073 + } + }, + "time": "0:03:09.746628", + "diversity": { + "train": { + "mean_centered_corr": 0.08831919729709625, + "mean_pairwise_disagreement": 0.041946592713174725, + "member_std": 0.03971056640148163 + }, + "val": { + "mean_centered_corr": 0.10093759745359421, + "mean_pairwise_disagreement": 0.040990251939039976, + "member_std": 0.044565752148628235 + }, + "test": { + "mean_centered_corr": 0.11717803031206131, + "mean_pairwise_disagreement": 0.04949915370834117, + "member_std": 0.04652492329478264 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 11, + 14, + 0, + 21, + 3, + 8, + 26, + 27, + 25, + 20, + 23, + 16, + 14 + ], + "head_selection_time": "0:00:13.167718" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..d33592680 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,309 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9485108852386475, + "min_features_per_member": 274.0, + "mean_features_per_member": 282.65625, + "max_features_per_member": 289.0, + "min_members_per_feature": 25.0, + "mean_members_per_feature": 30.35234832763672, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9021152843018349, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9466985010435772, + "recall": 0.9870748635771633, + "f1-score": 0.9664651617016812, + "support": 181971.0 + }, + "1": { + "precision": 0.9319995374118192, + "recall": 0.7611986115374625, + "f1-score": 0.8379843248372044, + "support": 42349.0 + }, + "accuracy": 0.9444320613409415, + "macro avg": { + "precision": 0.9393490192276982, + "recall": 0.8741367375573129, + "f1-score": 0.9022247432694428, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9439235081279151, + "recall": 0.9444320613409415, + "f1-score": 0.942209478033824, + "support": 224320.0 + }, + "cross-entropy": 0.13594152929391584, + "roc-auc": 0.9810912447120521, + "score": 0.9810912447120521 + }, + "val": { + "0": { + "precision": 0.9318143269067677, + "recall": 0.9844692670688728, + "f1-score": 0.9574183803880563, + "support": 16741.0 + }, + "1": { + "precision": 0.893920848633211, + "recall": 0.6449808654695319, + "f1-score": 0.7493160054719562, + "support": 3397.0 + }, + "accuracy": 0.9272023041016982, + "macro avg": { + "precision": 0.9128675877699893, + "recall": 0.8147250662692024, + "f1-score": 0.8533671929300062, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9254222251243032, + "recall": 0.9272023041016982, + "f1-score": 0.9223144094083169, + "support": 20138.0 + }, + "cross-entropy": 0.17569113158213984, + "roc-auc": 0.9590785004678369, + "score": 0.9590785004678369 + }, + "test": { + "0": { + "precision": 0.9279227331699582, + "recall": 0.9792348064197155, + "f1-score": 0.952888494134192, + "support": 13147.0 + }, + "1": { + "precision": 0.8872366790582404, + "recall": 0.6823379923761118, + "f1-score": 0.7714131801041479, + "support": 3148.0 + }, + "accuracy": 0.9218778766492789, + "macro avg": { + "precision": 0.9075797061140993, + "recall": 0.8307863993979137, + "f1-score": 0.86215083711917, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9200626719030857, + "recall": 0.9218778766492789, + "f1-score": 0.9178296240165744, + "support": 16295.0 + }, + "cross-entropy": 0.17937786610967274, + "roc-auc": 0.962897309467792, + "score": 0.962897309467792 + } + }, + "time": "0:03:06.503534", + "diversity": { + "train": { + "mean_centered_corr": 0.08432834595441818, + "mean_pairwise_disagreement": 0.03995795716660531, + "member_std": 0.04036388918757439 + }, + "val": { + "mean_centered_corr": 0.0944766253232956, + "mean_pairwise_disagreement": 0.03774069805439243, + "member_std": 0.043501291424036026 + }, + "test": { + "mean_centered_corr": 0.10949686914682388, + "mean_pairwise_disagreement": 0.04558554969365232, + "member_std": 0.04584404453635216 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 3, + 10, + 30, + 0, + 6, + 1, + 27, + 11, + 4, + 3, + 2 + ], + "head_selection_time": "0:00:16.177425" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7721a02fc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9413658167129513, + "recall": 0.9835633150337142, + "f1-score": 0.9620020478421065, + "support": 181971.0 + }, + "1": { + "precision": 0.9125233972859148, + "recall": 0.7367588372806914, + "f1-score": 0.8152754732757607, + "support": 42349.0 + }, + "accuracy": 0.9369695078459344, + "macro avg": { + "precision": 0.926944606999433, + "recall": 0.8601610761572027, + "f1-score": 0.8886387605589336, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9359207042828712, + "recall": 0.9369695078459344, + "f1-score": 0.9343017816763158, + "support": 224320.0 + }, + "cross-entropy": 0.14393086493300278, + "roc-auc": 0.9759690841627111, + "score": 0.9759690841627111 + }, + "val": { + "0": { + "precision": 0.9325333937061354, + "recall": 0.9841705991278896, + "f1-score": 0.9576564270975617, + "support": 16741.0 + }, + "1": { + "precision": 0.8927125506072875, + "recall": 0.6491021489549602, + "f1-score": 0.7516618373956026, + "support": 3397.0 + }, + "accuracy": 0.9276492203793822, + "macro avg": { + "precision": 0.9126229721567114, + "recall": 0.8166363740414249, + "f1-score": 0.8546591322465822, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.925816172333269, + "recall": 0.9276492203793822, + "f1-score": 0.9229080101138715, + "support": 20138.0 + }, + "cross-entropy": 0.17925081582806562, + "roc-auc": 0.9560453459701026, + "score": 0.9560453459701026 + }, + "test": { + "0": { + "precision": 0.9250304899921085, + "recall": 0.9807560660226667, + "f1-score": 0.9520785645721037, + "support": 13147.0 + }, + "1": { + "precision": 0.8926146010186757, + "recall": 0.6680432020330368, + "f1-score": 0.764171511627907, + "support": 3148.0 + }, + "accuracy": 0.9203436637005217, + "macro avg": { + "precision": 0.9088225455053921, + "recall": 0.8243996340278518, + "f1-score": 0.8581250381000054, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9187681261695637, + "recall": 0.9203436637005217, + "f1-score": 0.9157771590692911, + "support": 16295.0 + }, + "cross-entropy": 0.18465226452577868, + "roc-auc": 0.9600924508313724, + "score": 0.9600924508313724 + } + }, + "time": "0:06:58.103811", + "diversity": { + "train": { + "mean_centered_corr": 0.05266324058175087, + "mean_pairwise_disagreement": 0.03131294111810317, + "member_std": 0.028657343238592148 + }, + "val": { + "mean_centered_corr": 0.06352649629116058, + "mean_pairwise_disagreement": 0.02856399712948398, + "member_std": 0.030144035816192627 + }, + "test": { + "mean_centered_corr": 0.0760873481631279, + "mean_pairwise_disagreement": 0.03589971691296558, + "member_std": 0.032248713076114655 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.778477" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..1fc960885 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/1/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9482846874900025, + "recall": 0.9773370482109787, + "f1-score": 0.9625917075533738, + "support": 181971.0 + }, + "1": { + "precision": 0.8878555501169304, + "recall": 0.770974521240171, + "f1-score": 0.8252973218912326, + "support": 42349.0 + }, + "accuracy": 0.938378209700428, + "macro avg": { + "precision": 0.9180701188034665, + "recall": 0.8741557847255748, + "f1-score": 0.8939445147223032, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9368763710732219, + "recall": 0.938378209700428, + "f1-score": 0.9366721286553442, + "support": 224320.0 + }, + "cross-entropy": 0.1408223018750539, + "roc-auc": 0.9772865244821658, + "score": 0.9772865244821658 + }, + "val": { + "0": { + "precision": 0.9336525307797537, + "recall": 0.9784361746610118, + "f1-score": 0.9555199066647222, + "support": 16741.0 + }, + "1": { + "precision": 0.8608326908249807, + "recall": 0.6573447159258169, + "f1-score": 0.7454515105992322, + "support": 3397.0 + }, + "accuracy": 0.9242725196146588, + "macro avg": { + "precision": 0.8972426108023672, + "recall": 0.8178904452934144, + "f1-score": 0.8504857086319773, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9213688384405759, + "recall": 0.9242725196146588, + "f1-score": 0.9200842953113372, + "support": 20138.0 + }, + "cross-entropy": 0.18036337330439542, + "roc-auc": 0.9560549117846388, + "score": 0.9560549117846388 + }, + "test": { + "0": { + "precision": 0.9309239288837073, + "recall": 0.9717806343652544, + "f1-score": 0.9509136243533921, + "support": 13147.0 + }, + "1": { + "precision": 0.8556981719175418, + "recall": 0.6988564167725541, + "f1-score": 0.7693652736492393, + "support": 3148.0 + }, + "accuracy": 0.9190549248235655, + "macro avg": { + "precision": 0.8933110504006245, + "recall": 0.8353185255689042, + "f1-score": 0.8601394490013157, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9163912082375282, + "recall": 0.9190549248235655, + "f1-score": 0.9158406444198742, + "support": 16295.0 + }, + "cross-entropy": 0.18712658329229517, + "roc-auc": 0.9591263205069757, + "score": 0.9591263205069757 + } + }, + "time": "0:06:59.217282", + "diversity": { + "train": { + "mean_centered_corr": 0.04887603968381882, + "mean_pairwise_disagreement": 0.031215747736528462, + "member_std": 0.028810881078243256 + }, + "val": { + "mean_centered_corr": 0.053518619388341904, + "mean_pairwise_disagreement": 0.027906437516619198, + "member_std": 0.030356688424944878 + }, + "test": { + "mean_centered_corr": 0.06887096166610718, + "mean_pairwise_disagreement": 0.035209073632323395, + "member_std": 0.032625459134578705 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 14 + ], + "head_selection_time": "0:00:00.940553" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..e1d98b16e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-best-head-evaluation/2/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.942254416998281, + "recall": 0.9850250864148683, + "f1-score": 0.9631651630028856, + "support": 181971.0 + }, + "1": { + "precision": 0.920062190149315, + "recall": 0.7406078065597771, + "f1-score": 0.8206389492137418, + "support": 42349.0 + }, + "accuracy": 0.9388819543509273, + "macro avg": { + "precision": 0.931158303573798, + "recall": 0.8628164464873227, + "f1-score": 0.8919020561083137, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9380647833729829, + "recall": 0.9388819543509273, + "f1-score": 0.9362578759675947, + "support": 224320.0 + }, + "cross-entropy": 0.13956166211953258, + "roc-auc": 0.9780078991108505, + "score": 0.9780078991108505 + }, + "val": { + "0": { + "precision": 0.9299007444168734, + "recall": 0.9849471357744459, + "f1-score": 0.9566327270617585, + "support": 16741.0 + }, + "1": { + "precision": 0.8952618453865336, + "recall": 0.6340889019723285, + "f1-score": 0.7423746338100983, + "support": 3397.0 + }, + "accuracy": 0.9257622405402721, + "macro avg": { + "precision": 0.9125812949017036, + "recall": 0.8095180188733873, + "f1-score": 0.8495036804359284, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9240576448038997, + "recall": 0.9257622405402721, + "f1-score": 0.9204903721717053, + "support": 20138.0 + }, + "cross-entropy": 0.19004422805681795, + "roc-auc": 0.9556372338569274, + "score": 0.9556372338569274 + }, + "test": { + "0": { + "precision": 0.9235277301315037, + "recall": 0.9828858294667985, + "f1-score": 0.9522826928037142, + "support": 13147.0 + }, + "1": { + "precision": 0.9023013460703431, + "recall": 0.6601016518424396, + "f1-score": 0.7624289121262153, + "support": 3148.0 + }, + "accuracy": 0.9205277692543725, + "macro avg": { + "precision": 0.9129145381009234, + "recall": 0.821493740654619, + "f1-score": 0.8573558024649648, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9194270454414434, + "recall": 0.9205277692543725, + "f1-score": 0.9156052026795801, + "support": 16295.0 + }, + "cross-entropy": 0.19223543365810944, + "roc-auc": 0.9608528752531366, + "score": 0.9608528752531366 + } + }, + "time": "0:07:13.032423", + "diversity": { + "train": { + "mean_centered_corr": 0.05698179081082344, + "mean_pairwise_disagreement": 0.03173294702843864, + "member_std": 0.031025612726807594 + }, + "val": { + "mean_centered_corr": 0.06959165632724762, + "mean_pairwise_disagreement": 0.030371885602247718, + "member_std": 0.033528175204992294 + }, + "test": { + "mean_centered_corr": 0.08432681858539581, + "mean_pairwise_disagreement": 0.03709145393896802, + "member_std": 0.03507966548204422 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1 + ], + "head_selection_time": "0:00:01.404215" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml new file mode 100644 index 000000000..601f75de8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0/report.json new file mode 100644 index 000000000..4bb9206ba --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9463105683240804, + "recall": 0.9844810436827847, + "f1-score": 0.9650185034555944, + "support": 181971.0 + }, + "1": { + "precision": 0.9193350281356223, + "recall": 0.7599943328059694, + "f1-score": 0.8321052767651698, + "support": 42349.0 + }, + "accuracy": 0.9421005706134095, + "macro avg": { + "precision": 0.9328227982298514, + "recall": 0.872237688244377, + "f1-score": 0.898561890110382, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9412179009228634, + "recall": 0.9421005706134095, + "f1-score": 0.9399260362787364, + "support": 224320.0 + }, + "cross-entropy": 0.1363573248497679, + "roc-auc": 0.9795959593437451, + "score": 0.9795959593437451 + }, + "val": { + "0": { + "precision": 0.9344513928368391, + "recall": 0.9818409891882205, + "f1-score": 0.9575602225393959, + "support": 16741.0 + }, + "1": { + "precision": 0.880690737833595, + "recall": 0.6605828672357963, + "f1-score": 0.7549201009251472, + "support": 3397.0 + }, + "accuracy": 0.9276492203793822, + "macro avg": { + "precision": 0.907571065335217, + "recall": 0.8212119282120084, + "f1-score": 0.8562401617322715, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9253827194310382, + "recall": 0.9276492203793822, + "f1-score": 0.9233776575814258, + "support": 20138.0 + }, + "cross-entropy": 0.17415890290359384, + "roc-auc": 0.9581626616470993, + "score": 0.9581626616470993 + }, + "test": { + "0": { + "precision": 0.9295326291419477, + "recall": 0.9772571689358789, + "f1-score": 0.9527976565686529, + "support": 13147.0 + }, + "1": { + "precision": 0.879094217549535, + "recall": 0.6905972045743329, + "f1-score": 0.7735278420209927, + "support": 3148.0 + }, + "accuracy": 0.9218778766492789, + "macro avg": { + "precision": 0.9043134233457413, + "recall": 0.8339271867551059, + "f1-score": 0.8631627492948228, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9197885285164236, + "recall": 0.9218778766492789, + "f1-score": 0.9181648626934744, + "support": 16295.0 + }, + "cross-entropy": 0.17842812985458048, + "roc-auc": 0.9620964494052155, + "score": 0.9620964494052155 + } + }, + "time": "0:06:58.103811", + "diversity": { + "train": { + "mean_centered_corr": 0.05266324058175087, + "mean_pairwise_disagreement": 0.03131294111810317, + "member_std": 0.028657343238592148 + }, + "val": { + "mean_centered_corr": 0.06352649629116058, + "mean_pairwise_disagreement": 0.02856399712948398, + "member_std": 0.030144035816192627 + }, + "test": { + "mean_centered_corr": 0.0760873481631279, + "mean_pairwise_disagreement": 0.03589971691296558, + "member_std": 0.032248713076114655 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml new file mode 100644 index 000000000..5643d56c1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1/report.json new file mode 100644 index 000000000..49f122420 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9447127920145382, + "recall": 0.9855911106714806, + "f1-score": 0.964719108376186, + "support": 181971.0 + }, + "1": { + "precision": 0.9239448875997099, + "recall": 0.7521547143970342, + "f1-score": 0.8292460689367905, + "support": 42349.0 + }, + "accuracy": 0.9415210413694722, + "macro avg": { + "precision": 0.9343288398071241, + "recall": 0.8688729125342574, + "f1-score": 0.8969825886564883, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9407920538544831, + "recall": 0.9415210413694722, + "f1-score": 0.9391433784046322, + "support": 224320.0 + }, + "cross-entropy": 0.13643410406106432, + "roc-auc": 0.9796283338071923, + "score": 0.9796283338071923 + }, + "val": { + "0": { + "precision": 0.9313542550183772, + "recall": 0.9838719311869064, + "f1-score": 0.9568930459536397, + "support": 16741.0 + }, + "1": { + "precision": 0.889930697105585, + "recall": 0.6426258463350015, + "f1-score": 0.7463247863247863, + "support": 3397.0 + }, + "accuracy": 0.9263084715463303, + "macro avg": { + "precision": 0.9106424760619811, + "recall": 0.8132488887609539, + "f1-score": 0.851608916139213, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9243666779883963, + "recall": 0.9263084715463303, + "f1-score": 0.9213731145821422, + "support": 20138.0 + }, + "cross-entropy": 0.17370455502811047, + "roc-auc": 0.9583407546059616, + "score": 0.9583407546059616 + }, + "test": { + "0": { + "precision": 0.9288657558013446, + "recall": 0.9773332319160265, + "f1-score": 0.9524833209785026, + "support": 13147.0 + }, + "1": { + "precision": 0.8789601949634444, + "recall": 0.6874205844980941, + "f1-score": 0.7714795008912656, + "support": 3148.0 + }, + "accuracy": 0.9213255599877263, + "macro avg": { + "precision": 0.9039129753823945, + "recall": 0.8323769082070602, + "f1-score": 0.8619814109348841, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9192245955977417, + "recall": 0.9213255599877263, + "f1-score": 0.9175155378772678, + "support": 16295.0 + }, + "cross-entropy": 0.17839301052302864, + "roc-auc": 0.9620763221935056, + "score": 0.9620763221935056 + } + }, + "time": "0:06:59.217282", + "diversity": { + "train": { + "mean_centered_corr": 0.04887603968381882, + "mean_pairwise_disagreement": 0.031215747736528462, + "member_std": 0.028810881078243256 + }, + "val": { + "mean_centered_corr": 0.053518619388341904, + "mean_pairwise_disagreement": 0.027906437516619198, + "member_std": 0.030356688424944878 + }, + "test": { + "mean_centered_corr": 0.06887096166610718, + "mean_pairwise_disagreement": 0.035209073632323395, + "member_std": 0.032625459134578705 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml new file mode 100644 index 000000000..0fdbfa723 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2/report.json new file mode 100644 index 000000000..58043ba69 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9463349547713352, + "recall": 0.9899929109583395, + "f1-score": 0.9676717597230466, + "support": 181971.0 + }, + "1": { + "precision": 0.946368616363315, + "recall": 0.7587664407660157, + "f1-score": 0.8422473559362017, + "support": 42349.0 + }, + "accuracy": 0.9463400499286734, + "macro avg": { + "precision": 0.9463517855673251, + "recall": 0.8743796758621776, + "f1-score": 0.9049595578296241, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9463413096873425, + "recall": 0.9463400499286734, + "f1-score": 0.943993094976394, + "support": 224320.0 + }, + "cross-entropy": 0.12561167365428139, + "roc-auc": 0.9837216635411232, + "score": 0.9837216635411232 + }, + "val": { + "0": { + "precision": 0.9312468222134342, + "recall": 0.9846484678334627, + "f1-score": 0.9572034144358632, + "support": 16741.0 + }, + "1": { + "precision": 0.8945424702503078, + "recall": 0.6417427141595525, + "f1-score": 0.747343160781625, + "support": 3397.0 + }, + "accuracy": 0.9268050451882014, + "macro avg": { + "precision": 0.912894646231871, + "recall": 0.8131955909965076, + "f1-score": 0.8522732876087441, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9250553094704239, + "recall": 0.9268050451882014, + "f1-score": 0.9218029138070297, + "support": 20138.0 + }, + "cross-entropy": 0.17572658501355687, + "roc-auc": 0.9581674973070209, + "score": 0.9581674973070209 + }, + "test": { + "0": { + "precision": 0.9270114942528735, + "recall": 0.9815166958241424, + "f1-score": 0.9534857945099199, + "support": 13147.0 + }, + "1": { + "precision": 0.8976842105263158, + "recall": 0.6772554002541296, + "f1-score": 0.7720441788882854, + "support": 3148.0 + }, + "accuracy": 0.922737035900583, + "macro avg": { + "precision": 0.9123478523895947, + "recall": 0.829386048039136, + "f1-score": 0.8627649866991026, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9213458121926585, + "recall": 0.922737035900583, + "f1-score": 0.9184334345236109, + "support": 16295.0 + }, + "cross-entropy": 0.17887720986431865, + "roc-auc": 0.9627480104988175, + "score": 0.9627480104988175 + } + }, + "time": "0:07:13.032423", + "diversity": { + "train": { + "mean_centered_corr": 0.05698179081082344, + "mean_pairwise_disagreement": 0.03173294702843864, + "member_std": 0.031025612726807594 + }, + "val": { + "mean_centered_corr": 0.06959165632724762, + "mean_pairwise_disagreement": 0.030371885602247718, + "member_std": 0.033528175204992294 + }, + "test": { + "mean_centered_corr": 0.08432681858539581, + "mean_pairwise_disagreement": 0.03709145393896802, + "member_std": 0.03507966548204422 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b14bbaa40 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,284 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9441162844567987, + "recall": 0.985503184573366, + "f1-score": 0.9643658968753949, + "support": 181971.0 + }, + "1": { + "precision": 0.923251483765856, + "recall": 0.749344730690217, + "f1-score": 0.8272572046766856, + "support": 42349.0 + }, + "accuracy": 0.9409192225392297, + "macro avg": { + "precision": 0.9336838841113273, + "recall": 0.8674239576317915, + "f1-score": 0.8958115507760402, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9401772534098091, + "recall": 0.9409192225392297, + "f1-score": 0.9384813747377204, + "support": 224320.0 + }, + "cross-entropy": 0.13816220002148916, + "roc-auc": 0.9787552331834622, + "score": 0.9787552331834622 + }, + "val": { + "0": { + "precision": 0.931622965641953, + "recall": 0.984767935009856, + "f1-score": 0.9574585474925226, + "support": 16741.0 + }, + "1": { + "precision": 0.8955773955773956, + "recall": 0.6438033559022667, + "f1-score": 0.7491008734372324, + "support": 3397.0 + }, + "accuracy": 0.9272519614658854, + "macro avg": { + "precision": 0.9136001806096743, + "recall": 0.8142856454560614, + "f1-score": 0.8532797104648775, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.925542580225859, + "recall": 0.9272519614658854, + "f1-score": 0.9223115111053034, + "support": 20138.0 + }, + "cross-entropy": 0.17350924991795408, + "roc-auc": 0.9591329763748824, + "score": 0.9591329763748824 + }, + "test": { + "0": { + "precision": 0.9269169903610991, + "recall": 0.9801475621814862, + "f1-score": 0.9527893822322452, + "support": 13147.0 + }, + "1": { + "precision": 0.8909318846636022, + "recall": 0.6772554002541296, + "f1-score": 0.769536184804187, + "support": 3148.0 + }, + "accuracy": 0.9216324025774778, + "macro avg": { + "precision": 0.9089244375123506, + "recall": 0.828701481217808, + "f1-score": 0.8611627835182161, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9199650963607481, + "recall": 0.9216324025774778, + "f1-score": 0.9173870462087087, + "support": 16295.0 + }, + "cross-entropy": 0.17845179542990083, + "roc-auc": 0.962370812537228, + "score": 0.962370812537228 + } + }, + "time": "0:06:58.103811", + "diversity": { + "train": { + "mean_centered_corr": 0.05266324058175087, + "mean_pairwise_disagreement": 0.03131294111810317, + "member_std": 0.028657343238592148 + }, + "val": { + "mean_centered_corr": 0.06352649629116058, + "mean_pairwise_disagreement": 0.02856399712948398, + "member_std": 0.030144035816192627 + }, + "test": { + "mean_centered_corr": 0.0760873481631279, + "mean_pairwise_disagreement": 0.03589971691296558, + "member_std": 0.032248713076114655 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 24, + 30, + 13, + 17, + 21, + 28, + 24, + 6, + 8 + ], + "head_selection_time": "0:00:10.755431" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..24ba88a83 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,284 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9462082666286251, + "recall": 0.984145825433723, + "f1-score": 0.9648042495649691, + "support": 181971.0 + }, + "1": { + "precision": 0.917696060251619, + "recall": 0.7595929065621384, + "f1-score": 0.8311929924291362, + "support": 42349.0 + }, + "accuracy": 0.941752853067047, + "macro avg": { + "precision": 0.931952163440122, + "recall": 0.8718693659979306, + "f1-score": 0.8979986209970527, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9408254945714754, + "recall": 0.941752853067047, + "f1-score": 0.9395800023803874, + "support": 224320.0 + }, + "cross-entropy": 0.13658565047383905, + "roc-auc": 0.9794746002312951, + "score": 0.9794746002312951 + }, + "val": { + "0": { + "precision": 0.9323214589114799, + "recall": 0.9833343288931367, + "f1-score": 0.9571486714343858, + "support": 16741.0 + }, + "1": { + "precision": 0.8875453446191052, + "recall": 0.6482190167795113, + "f1-score": 0.7492344334807758, + "support": 3397.0 + }, + "accuracy": 0.9268050451882014, + "macro avg": { + "precision": 0.9099334017652925, + "recall": 0.815776672836324, + "f1-score": 0.8531915524575808, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9247683523343027, + "recall": 0.9268050451882014, + "f1-score": 0.9220764365387452, + "support": 20138.0 + }, + "cross-entropy": 0.17285360392280882, + "roc-auc": 0.9590165424760763, + "score": 0.9590165424760763 + }, + "test": { + "0": { + "precision": 0.9310144506571781, + "recall": 0.9752034684718948, + "f1-score": 0.952596775391931, + "support": 13147.0 + }, + "1": { + "precision": 0.8708399366085579, + "recall": 0.6982210927573063, + "f1-score": 0.7750352609308886, + "support": 3148.0 + }, + "accuracy": 0.921693771095428, + "macro avg": { + "precision": 0.900927193632868, + "recall": 0.8367122806146006, + "f1-score": 0.8638160181614098, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9193894509502093, + "recall": 0.921693771095428, + "f1-score": 0.9182940047553332, + "support": 16295.0 + }, + "cross-entropy": 0.17828631663015132, + "roc-auc": 0.9620755973239362, + "score": 0.9620755973239362 + } + }, + "time": "0:06:59.217282", + "diversity": { + "train": { + "mean_centered_corr": 0.04887603968381882, + "mean_pairwise_disagreement": 0.031215747736528462, + "member_std": 0.028810881078243256 + }, + "val": { + "mean_centered_corr": 0.053518619388341904, + "mean_pairwise_disagreement": 0.027906437516619198, + "member_std": 0.030356688424944878 + }, + "test": { + "mean_centered_corr": 0.06887096166610718, + "mean_pairwise_disagreement": 0.035209073632323395, + "member_std": 0.032625459134578705 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 14, + 26, + 5, + 21, + 13, + 24, + 18, + 30, + 28, + 16 + ], + "head_selection_time": "0:00:10.855848" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..ddb74823e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,293 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9461253448955459, + "recall": 0.9893004929356876, + "f1-score": 0.9672313470124595, + "support": 181971.0 + }, + "1": { + "precision": 0.9428109854604201, + "recall": 0.7579399749698931, + "f1-score": 0.840327774432547, + "support": 42349.0 + }, + "accuracy": 0.9456223252496434, + "macro avg": { + "precision": 0.9444681651779829, + "recall": 0.8736202339527903, + "f1-score": 0.9037795607225032, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9454996324859608, + "recall": 0.9456223252496434, + "f1-score": 0.9432734324476115, + "support": 224320.0 + }, + "cross-entropy": 0.12704980324137066, + "roc-auc": 0.9832599119647002, + "score": 0.9832599119647002 + }, + "val": { + "0": { + "precision": 0.9311162554344757, + "recall": 0.9850666029508393, + "f1-score": 0.9573319400905608, + "support": 16741.0 + }, + "1": { + "precision": 0.8969921714050267, + "recall": 0.6408595819841036, + "f1-score": 0.7475961538461539, + "support": 3397.0 + }, + "accuracy": 0.9270036746449498, + "macro avg": { + "precision": 0.9140542134197512, + "recall": 0.8129630924674714, + "f1-score": 0.8524640469683573, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9253599979387941, + "recall": 0.9270036746449498, + "f1-score": 0.921952435379455, + "support": 20138.0 + }, + "cross-entropy": 0.17486402172446866, + "roc-auc": 0.9587859694188998, + "score": 0.9587859694188998 + }, + "test": { + "0": { + "precision": 0.9278120959632237, + "recall": 0.9825055145660607, + "f1-score": 0.9543758542982748, + "support": 13147.0 + }, + "1": { + "precision": 0.9030762747576907, + "recall": 0.6807496823379924, + "f1-score": 0.7763086397391776, + "support": 3148.0 + }, + "accuracy": 0.92420988033139, + "macro avg": { + "precision": 0.9154441853604571, + "recall": 0.8316275984520265, + "f1-score": 0.8653422470187262, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.923033429798448, + "recall": 0.92420988033139, + "f1-score": 0.9199753884233416, + "support": 16295.0 + }, + "cross-entropy": 0.1776396237040874, + "roc-auc": 0.9632457301074769, + "score": 0.9632457301074769 + } + }, + "time": "0:07:13.032423", + "diversity": { + "train": { + "mean_centered_corr": 0.05698179081082344, + "mean_pairwise_disagreement": 0.03173294702843864, + "member_std": 0.031025612726807594 + }, + "val": { + "mean_centered_corr": 0.06959165632724762, + "mean_pairwise_disagreement": 0.030371885602247718, + "member_std": 0.033528175204992294 + }, + "test": { + "mean_centered_corr": 0.08432681858539581, + "mean_pairwise_disagreement": 0.03709145393896802, + "member_std": 0.03507966548204422 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1, + 6, + 15, + 14, + 23, + 5, + 4, + 28, + 10, + 25, + 11, + 6, + 0, + 13, + 15, + 30, + 4, + 23, + 6 + ], + "head_selection_time": "0:00:26.623499" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..546302ba0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,190 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22658863410177663, + "mae": 0.12739618122577667, + "r2": 0.6719876527786255, + "score": -0.22658863410177663 + }, + "val": { + "rmse": 0.23955454428788905, + "mae": 0.13915589451789856, + "r2": 0.6166812181472778, + "score": -0.23955454428788905 + }, + "test": { + "rmse": 0.26588264760298186, + "mae": 0.1863766759634018, + "r2": 0.5169194340705872, + "score": -0.26588264760298186 + } + }, + "time": "0:00:48.800887", + "diversity": { + "train": { + "mean_centered_corr": -0.018623020499944687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05047355964779854 + }, + "val": { + "mean_centered_corr": -0.01870153285562992, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06864207983016968 + }, + "test": { + "mean_centered_corr": -0.019026711583137512, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09529124945402145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015244" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/1/report.json new file mode 100644 index 000000000..ace612a7a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/1/report.json @@ -0,0 +1,190 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 962, + "metrics": { + "train": { + "rmse": 0.2225609541964927, + "mae": 0.13133738934993744, + "r2": 0.6835451126098633, + "score": -0.2225609541964927 + }, + "val": { + "rmse": 0.24150787640249508, + "mae": 0.13435126841068268, + "r2": 0.6104046106338501, + "score": -0.24150787640249508 + }, + "test": { + "rmse": 0.2416341058127328, + "mae": 0.15234874188899994, + "r2": 0.6010154485702515, + "score": -0.2416341058127328 + } + }, + "time": "0:00:53.667022", + "diversity": { + "train": { + "mean_centered_corr": -0.017433883622288704, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049763333052396774 + }, + "val": { + "mean_centered_corr": -0.01871240884065628, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08149437606334686 + }, + "test": { + "mean_centered_corr": -0.018136458471417427, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11388187855482101 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28 + ], + "head_selection_time": "0:00:00.015286" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/2/report.json new file mode 100644 index 000000000..a4d197bb5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/2/report.json @@ -0,0 +1,190 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23639802450731898, + "mae": 0.12933363020420074, + "r2": 0.6429725885391235, + "score": -0.23639802450731898 + }, + "val": { + "rmse": 0.23944956044068919, + "mae": 0.13556918501853943, + "r2": 0.6170171499252319, + "score": -0.23944956044068919 + }, + "test": { + "rmse": 0.24308412455882378, + "mae": 0.15950754284858704, + "r2": 0.5962125658988953, + "score": -0.24308412455882378 + } + }, + "time": "0:00:45.117774", + "diversity": { + "train": { + "mean_centered_corr": -0.019429348409175873, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0477914959192276 + }, + "val": { + "mean_centered_corr": -0.017664093524217606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06744762510061264 + }, + "test": { + "mean_centered_corr": -0.018886378034949303, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07553428411483765 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 30 + ], + "head_selection_time": "0:00:00.015062" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..a68e82651 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml @@ -0,0 +1,34 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..c4a6eed6f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0/report.json @@ -0,0 +1,186 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22224988999004666, + "mae": 0.11853151023387909, + "r2": 0.6844290494918823, + "score": -0.22224988999004666 + }, + "val": { + "rmse": 0.23452539387100468, + "mae": 0.12745676934719086, + "r2": 0.6326068639755249, + "score": -0.23452539387100468 + }, + "test": { + "rmse": 0.23846659721529556, + "mae": 0.15488465130329132, + "r2": 0.6114072799682617, + "score": -0.23846659721529556 + } + }, + "time": "0:00:48.800887", + "diversity": { + "train": { + "mean_centered_corr": -0.018623020499944687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05047355964779854 + }, + "val": { + "mean_centered_corr": -0.01870153285562992, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06864207983016968 + }, + "test": { + "mean_centered_corr": -0.019026711583137512, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09529124945402145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml new file mode 100644 index 000000000..b6864e8d9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml @@ -0,0 +1,34 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1/report.json new file mode 100644 index 000000000..59ef4bb78 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1/report.json @@ -0,0 +1,186 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 962, + "metrics": { + "train": { + "rmse": 0.21503383723674574, + "mae": 0.11624167859554291, + "r2": 0.7045884132385254, + "score": -0.21503383723674574 + }, + "val": { + "rmse": 0.23623006536000674, + "mae": 0.1255212128162384, + "r2": 0.627246618270874, + "score": -0.23623006536000674 + }, + "test": { + "rmse": 0.2318497676950212, + "mae": 0.13931861519813538, + "r2": 0.632672905921936, + "score": -0.2318497676950212 + } + }, + "time": "0:00:53.667022", + "diversity": { + "train": { + "mean_centered_corr": -0.017433883622288704, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049763333052396774 + }, + "val": { + "mean_centered_corr": -0.01871240884065628, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08149437606334686 + }, + "test": { + "mean_centered_corr": -0.018136458471417427, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11388187855482101 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml new file mode 100644 index 000000000..e02a76aa5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml @@ -0,0 +1,34 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2/report.json new file mode 100644 index 000000000..97b454f47 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2/report.json @@ -0,0 +1,186 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23231268970798877, + "mae": 0.12570486962795258, + "r2": 0.6552059650421143, + "score": -0.23231268970798877 + }, + "val": { + "rmse": 0.23581900020763122, + "mae": 0.12935329973697662, + "r2": 0.6285427808761597, + "score": -0.23581900020763122 + }, + "test": { + "rmse": 0.23293721743587778, + "mae": 0.1457630842924118, + "r2": 0.6292190551757812, + "score": -0.23293721743587778 + } + }, + "time": "0:00:45.117774", + "diversity": { + "train": { + "mean_centered_corr": -0.019429348409175873, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0477914959192276 + }, + "val": { + "mean_centered_corr": -0.017664093524217606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06744762510061264 + }, + "test": { + "mean_centered_corr": -0.018886378034949303, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07553428411483765 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d2edb64f2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22302489965783162, + "mae": 0.11715596914291382, + "r2": 0.6822243928909302, + "score": -0.22302489965783162 + }, + "val": { + "rmse": 0.23257941028161666, + "mae": 0.12573255598545074, + "r2": 0.6386785507202148, + "score": -0.23257941028161666 + }, + "test": { + "rmse": 0.23549883338311248, + "mae": 0.14942243695259094, + "r2": 0.6210193037986755, + "score": -0.23549883338311248 + } + }, + "time": "0:00:48.800887", + "diversity": { + "train": { + "mean_centered_corr": -0.018623020499944687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05047355964779854 + }, + "val": { + "mean_centered_corr": -0.01870153285562992, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06864207983016968 + }, + "test": { + "mean_centered_corr": -0.019026711583137512, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09529124945402145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 19, + 21, + 12, + 16, + 18, + 8, + 22, + 19, + 29 + ], + "head_selection_time": "0:00:00.158388" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..bbce78c6f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 962, + "metrics": { + "train": { + "rmse": 0.21477064276002938, + "mae": 0.1157485619187355, + "r2": 0.7053111791610718, + "score": -0.21477064276002938 + }, + "val": { + "rmse": 0.2341956008327552, + "mae": 0.12569019198417664, + "r2": 0.6336394548416138, + "score": -0.2341956008327552 + }, + "test": { + "rmse": 0.23089961816224172, + "mae": 0.13695372641086578, + "r2": 0.6356774568557739, + "score": -0.23089961816224172 + } + }, + "time": "0:00:53.667022", + "diversity": { + "train": { + "mean_centered_corr": -0.017433883622288704, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049763333052396774 + }, + "val": { + "mean_centered_corr": -0.01871240884065628, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08149437606334686 + }, + "test": { + "mean_centered_corr": -0.018136458471417427, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11388187855482101 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28, + 6, + 21, + 1, + 31, + 13, + 14, + 21, + 6, + 15, + 5 + ], + "head_selection_time": "0:00:00.175132" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..e34fa6bb4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2/report.json @@ -0,0 +1,201 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23317289428038254, + "mae": 0.12557387351989746, + "r2": 0.652647852897644, + "score": -0.23317289428038254 + }, + "val": { + "rmse": 0.23449523543792233, + "mae": 0.12881776690483093, + "r2": 0.6327013969421387, + "score": -0.23449523543792233 + }, + "test": { + "rmse": 0.23591672540617004, + "mae": 0.1523103564977646, + "r2": 0.6196731328964233, + "score": -0.23591672540617004 + } + }, + "time": "0:00:45.117774", + "diversity": { + "train": { + "mean_centered_corr": -0.019429348409175873, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0477914959192276 + }, + "val": { + "mean_centered_corr": -0.017664093524217606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06744762510061264 + }, + "test": { + "mean_centered_corr": -0.018886378034949303, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07553428411483765 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 30, + 27, + 9, + 4, + 31, + 5, + 12, + 2, + 22, + 13, + 2, + 4 + ], + "head_selection_time": "0:00:00.188219" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bfcad143f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/0/report.json @@ -0,0 +1,643 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24287177140617242, + "mae": 0.12792739272117615, + "r2": 0.6231504678726196, + "score": -0.24287177140617242 + }, + "val": { + "rmse": 0.2388197411165918, + "mae": 0.13359415531158447, + "r2": 0.61902916431427, + "score": -0.2388197411165918 + }, + "test": { + "rmse": 0.2462523116085554, + "mae": 0.16663780808448792, + "r2": 0.5856186151504517, + "score": -0.2462523116085554 + } + }, + "time": "0:01:03.015496", + "diversity": { + "train": { + "mean_centered_corr": -0.018840299919247627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04810040444135666 + }, + "val": { + "mean_centered_corr": -0.020352300256490707, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061659909784793854 + }, + "test": { + "mean_centered_corr": -0.019725581631064415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08170976489782333 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015354" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..868c3077d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/1/report.json @@ -0,0 +1,643 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23314741034598338, + "mae": 0.12273190170526505, + "r2": 0.6527237892150879, + "score": -0.23314741034598338 + }, + "val": { + "rmse": 0.24061445857155628, + "mae": 0.13311389088630676, + "r2": 0.6132817268371582, + "score": -0.24061445857155628 + }, + "test": { + "rmse": 0.23970045288478414, + "mae": 0.14821729063987732, + "r2": 0.6073756217956543, + "score": -0.23970045288478414 + } + }, + "time": "0:01:09.119012", + "diversity": { + "train": { + "mean_centered_corr": -0.018567770719528198, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0464579202234745 + }, + "val": { + "mean_centered_corr": -0.01797967404127121, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06989888846874237 + }, + "test": { + "mean_centered_corr": -0.01808270625770092, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09372883290052414 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27 + ], + "head_selection_time": "0:00:00.015071" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..d12d59a2d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-best-head-evaluation/2/report.json @@ -0,0 +1,643 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 518, + "metrics": { + "train": { + "rmse": 0.24266637864500767, + "mae": 0.1298421323299408, + "r2": 0.6237875819206238, + "score": -0.24266637864500767 + }, + "val": { + "rmse": 0.2376592052308581, + "mae": 0.13035762310028076, + "r2": 0.6227228045463562, + "score": -0.2376592052308581 + }, + "test": { + "rmse": 0.23311093711112285, + "mae": 0.13708607852458954, + "r2": 0.6286658644676208, + "score": -0.23311093711112285 + } + }, + "time": "0:00:57.753537", + "diversity": { + "train": { + "mean_centered_corr": -0.019510088488459587, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044065676629543304 + }, + "val": { + "mean_centered_corr": -0.020489495247602463, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058154650032520294 + }, + "test": { + "mean_centered_corr": -0.02033652924001217, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06957374513149261 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27 + ], + "head_selection_time": "0:00:00.015257" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..523756861 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml @@ -0,0 +1,45 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..747fa815c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,639 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23279361488245573, + "mae": 0.12178773432970047, + "r2": 0.6537768840789795, + "score": -0.23279361488245573 + }, + "val": { + "rmse": 0.23523413233394516, + "mae": 0.12663741409778595, + "r2": 0.6303830146789551, + "score": -0.23523413233394516 + }, + "test": { + "rmse": 0.235149605552611, + "mae": 0.14915932714939117, + "r2": 0.6221424341201782, + "score": -0.235149605552611 + } + }, + "time": "0:01:03.015496", + "diversity": { + "train": { + "mean_centered_corr": -0.018840299919247627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04810040444135666 + }, + "val": { + "mean_centered_corr": -0.020352300256490707, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061659909784793854 + }, + "test": { + "mean_centered_corr": -0.019725581631064415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08170976489782333 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml new file mode 100644 index 000000000..66c960456 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml @@ -0,0 +1,45 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1/report.json new file mode 100644 index 000000000..5e3b9688f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1/report.json @@ -0,0 +1,639 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22716695623112323, + "mae": 0.11858279258012772, + "r2": 0.6703112125396729, + "score": -0.22716695623112323 + }, + "val": { + "rmse": 0.23639255622691643, + "mae": 0.12672214210033417, + "r2": 0.626733660697937, + "score": -0.23639255622691643 + }, + "test": { + "rmse": 0.2324256095265638, + "mae": 0.1419195681810379, + "r2": 0.6308460235595703, + "score": -0.2324256095265638 + } + }, + "time": "0:01:09.119012", + "diversity": { + "train": { + "mean_centered_corr": -0.018567770719528198, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0464579202234745 + }, + "val": { + "mean_centered_corr": -0.01797967404127121, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06989888846874237 + }, + "test": { + "mean_centered_corr": -0.01808270625770092, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09372883290052414 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml new file mode 100644 index 000000000..69535992d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml @@ -0,0 +1,45 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2/report.json new file mode 100644 index 000000000..f2a4c0d43 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2/report.json @@ -0,0 +1,639 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 518, + "metrics": { + "train": { + "rmse": 0.24112485206623196, + "mae": 0.12558841705322266, + "r2": 0.6285521388053894, + "score": -0.24112485206623196 + }, + "val": { + "rmse": 0.2359054032023901, + "mae": 0.12690992653369904, + "r2": 0.6282705068588257, + "score": -0.2359054032023901 + }, + "test": { + "rmse": 0.23245970634502372, + "mae": 0.14181140065193176, + "r2": 0.6307376623153687, + "score": -0.23245970634502372 + } + }, + "time": "0:00:57.753537", + "diversity": { + "train": { + "mean_centered_corr": -0.019510088488459587, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044065676629543304 + }, + "val": { + "mean_centered_corr": -0.020489495247602463, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058154650032520294 + }, + "test": { + "mean_centered_corr": -0.02033652924001217, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06957374513149261 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d85be1baa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,649 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23477394006764257, + "mae": 0.1248951405286789, + "r2": 0.6478613615036011, + "score": -0.23477394006764257 + }, + "val": { + "rmse": 0.23327114512588593, + "mae": 0.12512965500354767, + "r2": 0.6365260481834412, + "score": -0.23327114512588593 + }, + "test": { + "rmse": 0.23367141178994547, + "mae": 0.14663751423358917, + "r2": 0.6268780827522278, + "score": -0.23367141178994547 + } + }, + "time": "0:01:03.015496", + "diversity": { + "train": { + "mean_centered_corr": -0.018840299919247627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04810040444135666 + }, + "val": { + "mean_centered_corr": -0.020352300256490707, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061659909784793854 + }, + "test": { + "mean_centered_corr": -0.019725581631064415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08170976489782333 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 21, + 25, + 19, + 0, + 28 + ], + "head_selection_time": "0:00:00.116743" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..cba3c4596 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2278723424771184, + "mae": 0.11909302324056625, + "r2": 0.6682605743408203, + "score": -0.2278723424771184 + }, + "val": { + "rmse": 0.23463880474397839, + "mae": 0.12749379873275757, + "r2": 0.632251501083374, + "score": -0.23463880474397839 + }, + "test": { + "rmse": 0.2331063026442073, + "mae": 0.1443300098180771, + "r2": 0.6286805868148804, + "score": -0.2331063026442073 + } + }, + "time": "0:01:09.119012", + "diversity": { + "train": { + "mean_centered_corr": -0.018567770719528198, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0464579202234745 + }, + "val": { + "mean_centered_corr": -0.01797967404127121, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06989888846874237 + }, + "test": { + "mean_centered_corr": -0.01808270625770092, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09372883290052414 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27, + 1, + 19, + 31, + 14, + 1, + 6, + 29, + 16, + 31, + 4, + 1, + 1, + 3, + 29, + 19 + ], + "head_selection_time": "0:00:00.250837" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..e8f693d35 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 518, + "metrics": { + "train": { + "rmse": 0.24263932781325961, + "mae": 0.1284457892179489, + "r2": 0.6238714456558228, + "score": -0.24263932781325961 + }, + "val": { + "rmse": 0.23432768503071347, + "mae": 0.12745344638824463, + "r2": 0.6332260370254517, + "score": -0.23432768503071347 + }, + "test": { + "rmse": 0.2325014731401514, + "mae": 0.14458957314491272, + "r2": 0.6306049823760986, + "score": -0.2325014731401514 + } + }, + "time": "0:00:57.753537", + "diversity": { + "train": { + "mean_centered_corr": -0.019510088488459587, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044065676629543304 + }, + "val": { + "mean_centered_corr": -0.020489495247602463, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058154650032520294 + }, + "test": { + "mean_centered_corr": -0.02033652924001217, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06957374513149261 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27, + 9, + 22, + 5, + 12, + 30 + ], + "head_selection_time": "0:00:00.100965" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..27e477e2c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0/report.json @@ -0,0 +1,679 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.23359725177653518, + "mae": 0.12286694347858429, + "r2": 0.651382327079773, + "score": -0.23359725177653518 + }, + "val": { + "rmse": 0.23765498863853576, + "mae": 0.13634204864501953, + "r2": 0.6227362155914307, + "score": -0.23765498863853576 + }, + "test": { + "rmse": 0.2536206620016002, + "mae": 0.175298810005188, + "r2": 0.5604493618011475, + "score": -0.2536206620016002 + } + }, + "time": "0:01:22.331704", + "diversity": { + "train": { + "mean_centered_corr": -0.018672151491045952, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0502934493124485 + }, + "val": { + "mean_centered_corr": -0.01943311281502247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06968922913074493 + }, + "test": { + "mean_centered_corr": -0.020102957263588905, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09299962222576141 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015321" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1/report.json new file mode 100644 index 000000000..93cd7c674 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1/report.json @@ -0,0 +1,679 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9035395383834839, + "min_features_per_member": 338.0, + "mean_features_per_member": 354.1875, + "max_features_per_member": 365.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.913265228271484, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8235956120500022, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 592, + "metrics": { + "train": { + "rmse": 0.2391015457230593, + "mae": 0.12270964682102203, + "r2": 0.6347597241401672, + "score": -0.2391015457230593 + }, + "val": { + "rmse": 0.2374959967300924, + "mae": 0.12878940999507904, + "r2": 0.6232408285140991, + "score": -0.2374959967300924 + }, + "test": { + "rmse": 0.24912023719870632, + "mae": 0.166133850812912, + "r2": 0.5759104490280151, + "score": -0.24912023719870632 + } + }, + "time": "0:01:08.183283", + "diversity": { + "train": { + "mean_centered_corr": -0.019805658608675003, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046689730137586594 + }, + "val": { + "mean_centered_corr": -0.02030753530561924, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059032831341028214 + }, + "test": { + "mean_centered_corr": -0.019823072478175163, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07867440581321716 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 17 + ], + "head_selection_time": "0:00:00.015174" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2/report.json new file mode 100644 index 000000000..163a3a5e3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2/report.json @@ -0,0 +1,679 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9005899429321289, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.03125, + "max_features_per_member": 365.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.818878173828125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8198662012375459, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1036, + "metrics": { + "train": { + "rmse": 0.2242998063053038, + "mae": 0.12896357476711273, + "r2": 0.6785808801651001, + "score": -0.2242998063053038 + }, + "val": { + "rmse": 0.23866731604984137, + "mae": 0.1387324184179306, + "r2": 0.6195152997970581, + "score": -0.23866731604984137 + }, + "test": { + "rmse": 0.2357692652387237, + "mae": 0.14540770649909973, + "r2": 0.6201484203338623, + "score": -0.2357692652387237 + } + }, + "time": "0:01:25.772588", + "diversity": { + "train": { + "mean_centered_corr": -0.01864209584891796, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05142972990870476 + }, + "val": { + "mean_centered_corr": -0.018798964098095894, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0795377790927887 + }, + "test": { + "mean_centered_corr": -0.01805640198290348, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09804920852184296 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 30 + ], + "head_selection_time": "0:00:00.014937" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml new file mode 100644 index 000000000..e9f43e127 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml @@ -0,0 +1,58 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0/report.json new file mode 100644 index 000000000..78c9ce2d6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0/report.json @@ -0,0 +1,675 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22208876822485193, + "mae": 0.11730500310659409, + "r2": 0.6848864555358887, + "score": -0.22208876822485193 + }, + "val": { + "rmse": 0.23508488120210985, + "mae": 0.12783657014369965, + "r2": 0.6308518648147583, + "score": -0.23508488120210985 + }, + "test": { + "rmse": 0.23568789402876444, + "mae": 0.14989204704761505, + "r2": 0.6204105615615845, + "score": -0.23568789402876444 + } + }, + "time": "0:01:22.331704", + "diversity": { + "train": { + "mean_centered_corr": -0.018672151491045952, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0502934493124485 + }, + "val": { + "mean_centered_corr": -0.01943311281502247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06968922913074493 + }, + "test": { + "mean_centered_corr": -0.020102957263588905, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09299962222576141 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml new file mode 100644 index 000000000..c4ce180b9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml @@ -0,0 +1,58 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 1 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1/report.json new file mode 100644 index 000000000..b5d41c5d0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1/report.json @@ -0,0 +1,675 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9035395383834839, + "min_features_per_member": 338.0, + "mean_features_per_member": 354.1875, + "max_features_per_member": 365.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.913265228271484, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8235956120500022, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 592, + "metrics": { + "train": { + "rmse": 0.2377225313298363, + "mae": 0.12293801456689835, + "r2": 0.638960599899292, + "score": -0.2377225313298363 + }, + "val": { + "rmse": 0.23530641494725335, + "mae": 0.12682271003723145, + "r2": 0.6301558017730713, + "score": -0.23530641494725335 + }, + "test": { + "rmse": 0.23169352906704774, + "mae": 0.14042755961418152, + "r2": 0.6331678628921509, + "score": -0.23169352906704774 + } + }, + "time": "0:01:08.183283", + "diversity": { + "train": { + "mean_centered_corr": -0.019805658608675003, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046689730137586594 + }, + "val": { + "mean_centered_corr": -0.02030753530561924, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059032831341028214 + }, + "test": { + "mean_centered_corr": -0.019823072478175163, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07867440581321716 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml new file mode 100644 index 000000000..4f57370d4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml @@ -0,0 +1,58 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 2 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2/report.json new file mode 100644 index 000000000..d09b3432b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2/report.json @@ -0,0 +1,675 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9005899429321289, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.03125, + "max_features_per_member": 365.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.818878173828125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8198662012375459, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1036, + "metrics": { + "train": { + "rmse": 0.21596792187571293, + "mae": 0.11686611920595169, + "r2": 0.7020163536071777, + "score": -0.21596792187571293 + }, + "val": { + "rmse": 0.2356153727014129, + "mae": 0.1266695261001587, + "r2": 0.6291840076446533, + "score": -0.2356153727014129 + }, + "test": { + "rmse": 0.2360324823167294, + "mae": 0.15042589604854584, + "r2": 0.6192997694015503, + "score": -0.2360324823167294 + } + }, + "time": "0:01:25.772588", + "diversity": { + "train": { + "mean_centered_corr": -0.01864209584891796, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05142972990870476 + }, + "val": { + "mean_centered_corr": -0.018798964098095894, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0795377790927887 + }, + "test": { + "mean_centered_corr": -0.01805640198290348, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09804920852184296 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..635150edd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22343022528056483, + "mae": 0.11760932952165604, + "r2": 0.6810683012008667, + "score": -0.22343022528056483 + }, + "val": { + "rmse": 0.23320900639668657, + "mae": 0.12497496604919434, + "r2": 0.6367197036743164, + "score": -0.23320900639668657 + }, + "test": { + "rmse": 0.23362227205150313, + "mae": 0.14383481442928314, + "r2": 0.6270350217819214, + "score": -0.23362227205150313 + } + }, + "time": "0:01:22.331704", + "diversity": { + "train": { + "mean_centered_corr": -0.018672151491045952, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0502934493124485 + }, + "val": { + "mean_centered_corr": -0.01943311281502247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06968922913074493 + }, + "test": { + "mean_centered_corr": -0.020102957263588905, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09299962222576141 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 20, + 29, + 19, + 0, + 12, + 4, + 20, + 21, + 19 + ], + "head_selection_time": "0:00:00.159678" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..3117264d5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1/report.json @@ -0,0 +1,683 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9035395383834839, + "min_features_per_member": 338.0, + "mean_features_per_member": 354.1875, + "max_features_per_member": 365.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.913265228271484, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8235956120500022, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 592, + "metrics": { + "train": { + "rmse": 0.2359836526912841, + "mae": 0.12225823104381561, + "r2": 0.6442230939865112, + "score": -0.2359836526912841 + }, + "val": { + "rmse": 0.2332021853813015, + "mae": 0.12649986147880554, + "r2": 0.6367409229278564, + "score": -0.2332021853813015 + }, + "test": { + "rmse": 0.2324977398321572, + "mae": 0.1440209597349167, + "r2": 0.6306169033050537, + "score": -0.2324977398321572 + } + }, + "time": "0:01:08.183283", + "diversity": { + "train": { + "mean_centered_corr": -0.019805658608675003, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046689730137586594 + }, + "val": { + "mean_centered_corr": -0.02030753530561924, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059032831341028214 + }, + "test": { + "mean_centered_corr": -0.019823072478175163, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07867440581321716 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 17, + 27, + 2, + 15, + 28 + ], + "head_selection_time": "0:00:00.087511" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..d4ad57cf2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2/report.json @@ -0,0 +1,687 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9005899429321289, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.03125, + "max_features_per_member": 365.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.818878173828125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8198662012375459, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1036, + "metrics": { + "train": { + "rmse": 0.21572123708815089, + "mae": 0.11632093042135239, + "r2": 0.702696681022644, + "score": -0.21572123708815089 + }, + "val": { + "rmse": 0.23346459476611428, + "mae": 0.12624695897102356, + "r2": 0.635922908782959, + "score": -0.23346459476611428 + }, + "test": { + "rmse": 0.23881630936614148, + "mae": 0.15446454286575317, + "r2": 0.6102666854858398, + "score": -0.23881630936614148 + } + }, + "time": "0:01:25.772588", + "diversity": { + "train": { + "mean_centered_corr": -0.01864209584891796, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05142972990870476 + }, + "val": { + "mean_centered_corr": -0.018798964098095894, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0795377790927887 + }, + "test": { + "mean_centered_corr": -0.01805640198290348, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09804920852184296 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 30, + 2, + 24, + 19, + 20, + 9, + 15, + 27, + 2 + ], + "head_selection_time": "0:00:00.143383" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bc82de281 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/0/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2261742520907439, + "mae": 0.12224072217941284, + "r2": 0.6731863021850586, + "score": -0.2261742520907439 + }, + "val": { + "rmse": 0.2377136850270048, + "mae": 0.13327811658382416, + "r2": 0.622549831867218, + "score": -0.2377136850270048 + }, + "test": { + "rmse": 0.262345673404661, + "mae": 0.17990553379058838, + "r2": 0.5296865105628967, + "score": -0.262345673404661 + } + }, + "time": "0:01:27.301753", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015122" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..8e1a4d015 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/1/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2294608480279213, + "mae": 0.12013265490531921, + "r2": 0.6636192798614502, + "score": -0.2294608480279213 + }, + "val": { + "rmse": 0.2385787982165438, + "mae": 0.13280385732650757, + "r2": 0.6197975277900696, + "score": -0.2385787982165438 + }, + "test": { + "rmse": 0.24822352719647534, + "mae": 0.16104964911937714, + "r2": 0.5789579153060913, + "score": -0.24822352719647534 + } + }, + "time": "0:01:28.517490", + "diversity": { + "train": { + "mean_centered_corr": -0.01827217824757099, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04794318601489067 + }, + "val": { + "mean_centered_corr": -0.019240442663431168, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07059209793806076 + }, + "test": { + "mean_centered_corr": -0.020052773877978325, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09476596862077713 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1 + ], + "head_selection_time": "0:00:00.015630" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..e689e8acb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-best-head-evaluation/2/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22841502689654178, + "mae": 0.12657804787158966, + "r2": 0.6666785478591919, + "score": -0.22841502689654178 + }, + "val": { + "rmse": 0.24089231567126032, + "mae": 0.1354312300682068, + "r2": 0.612388014793396, + "score": -0.24089231567126032 + }, + "test": { + "rmse": 0.24860647860679874, + "mae": 0.16609469056129456, + "r2": 0.5776578187942505, + "score": -0.24860647860679874 + } + }, + "time": "0:01:27.073529", + "diversity": { + "train": { + "mean_centered_corr": -0.019437065348029137, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05152730643749237 + }, + "val": { + "mean_centered_corr": -0.018969101831316948, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07230502367019653 + }, + "test": { + "mean_centered_corr": -0.018757972866296768, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08676532655954361 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 30 + ], + "head_selection_time": "0:00:00.015302" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml new file mode 100644 index 000000000..05c49e679 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0/report.json new file mode 100644 index 000000000..5fee35a30 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0/report.json @@ -0,0 +1,196 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22265827445732783, + "mae": 0.11788544058799744, + "r2": 0.6832683086395264, + "score": -0.22265827445732783 + }, + "val": { + "rmse": 0.23474623366768377, + "mae": 0.1270996779203415, + "r2": 0.6319146752357483, + "score": -0.23474623366768377 + }, + "test": { + "rmse": 0.23936563514587833, + "mae": 0.15535929799079895, + "r2": 0.6084716320037842, + "score": -0.23936563514587833 + } + }, + "time": "0:01:27.301753", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml new file mode 100644 index 000000000..14067822a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml @@ -0,0 +1,44 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1/report.json new file mode 100644 index 000000000..d8fdf451a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1/report.json @@ -0,0 +1,196 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22373909069883033, + "mae": 0.11784303933382034, + "r2": 0.6801859140396118, + "score": -0.22373909069883033 + }, + "val": { + "rmse": 0.23723213436744667, + "mae": 0.12606672942638397, + "r2": 0.624077558517456, + "score": -0.23723213436744667 + }, + "test": { + "rmse": 0.23099161094856513, + "mae": 0.13413691520690918, + "r2": 0.635387122631073, + "score": -0.23099161094856513 + } + }, + "time": "0:01:28.517490", + "diversity": { + "train": { + "mean_centered_corr": -0.01827217824757099, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04794318601489067 + }, + "val": { + "mean_centered_corr": -0.019240442663431168, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07059209793806076 + }, + "test": { + "mean_centered_corr": -0.020052773877978325, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09476596862077713 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml new file mode 100644 index 000000000..0aaff1748 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml @@ -0,0 +1,44 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2/report.json new file mode 100644 index 000000000..d8eac53d6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2/report.json @@ -0,0 +1,196 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22303537248925367, + "mae": 0.12042270600795746, + "r2": 0.6821945309638977, + "score": -0.22303537248925367 + }, + "val": { + "rmse": 0.2356560348188032, + "mae": 0.12620244920253754, + "r2": 0.6290559768676758, + "score": -0.2356560348188032 + }, + "test": { + "rmse": 0.23118945759761478, + "mae": 0.1404353380203247, + "r2": 0.6347622871398926, + "score": -0.23118945759761478 + } + }, + "time": "0:01:27.073529", + "diversity": { + "train": { + "mean_centered_corr": -0.019437065348029137, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05152730643749237 + }, + "val": { + "mean_centered_corr": -0.018969101831316948, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07230502367019653 + }, + "test": { + "mean_centered_corr": -0.018757972866296768, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08676532655954361 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9ca33d310 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22450348389711117, + "mae": 0.11922673135995865, + "r2": 0.6779968738555908, + "score": -0.22450348389711117 + }, + "val": { + "rmse": 0.23295802298295287, + "mae": 0.1258699893951416, + "r2": 0.6375011801719666, + "score": -0.23295802298295287 + }, + "test": { + "rmse": 0.23903366020873654, + "mae": 0.15511320531368256, + "r2": 0.6095569133758545, + "score": -0.23903366020873654 + } + }, + "time": "0:01:27.301753", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 22, + 12, + 16, + 21, + 19, + 25, + 29 + ], + "head_selection_time": "0:00:00.128997" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..881467a21 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,206 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2241471801070519, + "mae": 0.11760734766721725, + "r2": 0.6790181398391724, + "score": -0.2241471801070519 + }, + "val": { + "rmse": 0.23392588182013857, + "mae": 0.12585178017616272, + "r2": 0.6344828009605408, + "score": -0.23392588182013857 + }, + "test": { + "rmse": 0.23227402448302542, + "mae": 0.14325864613056183, + "r2": 0.6313273906707764, + "score": -0.23227402448302542 + } + }, + "time": "0:01:28.517490", + "diversity": { + "train": { + "mean_centered_corr": -0.01827217824757099, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04794318601489067 + }, + "val": { + "mean_centered_corr": -0.019240442663431168, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07059209793806076 + }, + "test": { + "mean_centered_corr": -0.020052773877978325, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09476596862077713 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1, + 21, + 28, + 22, + 0, + 6, + 13 + ], + "head_selection_time": "0:00:00.124247" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..96f1d4f48 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,205 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2242032567429998, + "mae": 0.12045589089393616, + "r2": 0.6788575649261475, + "score": -0.2242032567429998 + }, + "val": { + "rmse": 0.23456056719125185, + "mae": 0.12557265162467957, + "r2": 0.63249671459198, + "score": -0.23456056719125185 + }, + "test": { + "rmse": 0.23191054380220286, + "mae": 0.14388112723827362, + "r2": 0.6324803829193115, + "score": -0.23191054380220286 + } + }, + "time": "0:01:27.073529", + "diversity": { + "train": { + "mean_centered_corr": -0.019437065348029137, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05152730643749237 + }, + "val": { + "mean_centered_corr": -0.018969101831316948, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07230502367019653 + }, + "test": { + "mean_centered_corr": -0.018757972866296768, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08676532655954361 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 30, + 12, + 5, + 27, + 9, + 15 + ], + "head_selection_time": "0:00:00.106322" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..688b587ad --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/0/report.json @@ -0,0 +1,226 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8038105964660645, + "min_features_per_member": 298.0, + "mean_features_per_member": 315.09375, + "max_features_per_member": 335.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.721939086914062, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6715097166219348, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2316786318343145, + "mae": 0.12621846795082092, + "r2": 0.6570855379104614, + "score": -0.2316786318343145 + }, + "val": { + "rmse": 0.2380346366558266, + "mae": 0.1380215585231781, + "r2": 0.6215299367904663, + "score": -0.2380346366558266 + }, + "test": { + "rmse": 0.2579307429494123, + "mae": 0.17571206390857697, + "r2": 0.5453828573226929, + "score": -0.2579307429494123 + } + }, + "time": "0:00:59.854964", + "diversity": { + "train": { + "mean_centered_corr": -0.017583154141902924, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05003640055656433 + }, + "val": { + "mean_centered_corr": -0.018760371953248978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07571490854024887 + }, + "test": { + "mean_centered_corr": -0.019624220207333565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09894119203090668 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.016145" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..6f8453216 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/1/report.json @@ -0,0 +1,226 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.8027742505073547, + "min_features_per_member": 296.0, + "mean_features_per_member": 314.6875, + "max_features_per_member": 330.0, + "min_members_per_feature": 18.0, + "mean_members_per_feature": 25.68877601623535, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6710553952491272, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 592, + "metrics": { + "train": { + "rmse": 0.24658122298608637, + "mae": 0.13536591827869415, + "r2": 0.6115511059761047, + "score": -0.24658122298608637 + }, + "val": { + "rmse": 0.2385396806698102, + "mae": 0.13372096419334412, + "r2": 0.6199221611022949, + "score": -0.2385396806698102 + }, + "test": { + "rmse": 0.2407010051283554, + "mae": 0.15062788128852844, + "r2": 0.6040909290313721, + "score": -0.2407010051283554 + } + }, + "time": "0:00:52.144209", + "diversity": { + "train": { + "mean_centered_corr": -0.01851801574230194, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785211756825447 + }, + "val": { + "mean_centered_corr": -0.019852403551340103, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0691390112042427 + }, + "test": { + "mean_centered_corr": -0.019887704402208328, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08872178196907043 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1 + ], + "head_selection_time": "0:00:00.015140" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..895187baa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-best-head-evaluation/2/report.json @@ -0,0 +1,226 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8008609414100647, + "min_features_per_member": 293.0, + "mean_features_per_member": 313.9375, + "max_features_per_member": 331.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.62755012512207, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6674885740537666, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1110, + "metrics": { + "train": { + "rmse": 0.22528715287379555, + "mae": 0.12278623133897781, + "r2": 0.6757449507713318, + "score": -0.22528715287379555 + }, + "val": { + "rmse": 0.24181054374010003, + "mae": 0.1348293274641037, + "r2": 0.6094274520874023, + "score": -0.24181054374010003 + }, + "test": { + "rmse": 0.24863506025450496, + "mae": 0.16692666709423065, + "r2": 0.5775606632232666, + "score": -0.24863506025450496 + } + }, + "time": "0:01:06.264634", + "diversity": { + "train": { + "mean_centered_corr": -0.01694483496248722, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0545165091753006 + }, + "val": { + "mean_centered_corr": -0.015533894300460815, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08842652291059494 + }, + "test": { + "mean_centered_corr": -0.016092214733362198, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11407508701086044 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 2 + ], + "head_selection_time": "0:00:00.014901" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml new file mode 100644 index 000000000..d44bc73bf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.8 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..66077afb4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0/report.json @@ -0,0 +1,222 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8038105964660645, + "min_features_per_member": 298.0, + "mean_features_per_member": 315.09375, + "max_features_per_member": 335.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.721939086914062, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6715097166219348, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22081238228152064, + "mae": 0.11986075341701508, + "r2": 0.6884980797767639, + "score": -0.22081238228152064 + }, + "val": { + "rmse": 0.2341279953511937, + "mae": 0.12804140150547028, + "r2": 0.6338509321212769, + "score": -0.2341279953511937 + }, + "test": { + "rmse": 0.23402994505001048, + "mae": 0.14647243916988373, + "r2": 0.6257321834564209, + "score": -0.23402994505001048 + } + }, + "time": "0:00:59.854964", + "diversity": { + "train": { + "mean_centered_corr": -0.017583154141902924, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05003640055656433 + }, + "val": { + "mean_centered_corr": -0.018760371953248978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07571490854024887 + }, + "test": { + "mean_centered_corr": -0.019624220207333565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09894119203090668 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml new file mode 100644 index 000000000..eba42670e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml @@ -0,0 +1,47 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.8 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 1 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1/report.json new file mode 100644 index 000000000..2ca6ea33a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1/report.json @@ -0,0 +1,222 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.8027742505073547, + "min_features_per_member": 296.0, + "mean_features_per_member": 314.6875, + "max_features_per_member": 330.0, + "min_members_per_feature": 18.0, + "mean_members_per_feature": 25.68877601623535, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6710553952491272, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 592, + "metrics": { + "train": { + "rmse": 0.24114945435053933, + "mae": 0.12663434445858002, + "r2": 0.6284763813018799, + "score": -0.24114945435053933 + }, + "val": { + "rmse": 0.23621923128975542, + "mae": 0.12754125893115997, + "r2": 0.6272808313369751, + "score": -0.23621923128975542 + }, + "test": { + "rmse": 0.23276193568912745, + "mae": 0.14217989146709442, + "r2": 0.6297768950462341, + "score": -0.23276193568912745 + } + }, + "time": "0:00:52.144209", + "diversity": { + "train": { + "mean_centered_corr": -0.01851801574230194, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785211756825447 + }, + "val": { + "mean_centered_corr": -0.019852403551340103, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0691390112042427 + }, + "test": { + "mean_centered_corr": -0.019887704402208328, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08872178196907043 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml new file mode 100644 index 000000000..c8868541f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml @@ -0,0 +1,47 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.8 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 2 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2/report.json new file mode 100644 index 000000000..b9db155ac --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2/report.json @@ -0,0 +1,222 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8008609414100647, + "min_features_per_member": 293.0, + "mean_features_per_member": 313.9375, + "max_features_per_member": 331.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.62755012512207, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6674885740537666, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1110, + "metrics": { + "train": { + "rmse": 0.21154753458799463, + "mae": 0.11532945930957794, + "r2": 0.7140896916389465, + "score": -0.21154753458799463 + }, + "val": { + "rmse": 0.23605616345434166, + "mae": 0.12597376108169556, + "r2": 0.6277952194213867, + "score": -0.23605616345434166 + }, + "test": { + "rmse": 0.23451195530114213, + "mae": 0.14579595625400543, + "r2": 0.6241888999938965, + "score": -0.23451195530114213 + } + }, + "time": "0:01:06.264634", + "diversity": { + "train": { + "mean_centered_corr": -0.01694483496248722, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0545165091753006 + }, + "val": { + "mean_centered_corr": -0.015533894300460815, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08842652291059494 + }, + "test": { + "mean_centered_corr": -0.016092214733362198, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11407508701086044 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b613a292e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,235 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8038105964660645, + "min_features_per_member": 298.0, + "mean_features_per_member": 315.09375, + "max_features_per_member": 335.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.721939086914062, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6715097166219348, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22132344126163775, + "mae": 0.11854095757007599, + "r2": 0.6870545148849487, + "score": -0.22132344126163775 + }, + "val": { + "rmse": 0.23237596587359466, + "mae": 0.1258111149072647, + "r2": 0.639310359954834, + "score": -0.23237596587359466 + }, + "test": { + "rmse": 0.23064696162155762, + "mae": 0.1411736011505127, + "r2": 0.6364743709564209, + "score": -0.23064696162155762 + } + }, + "time": "0:00:59.854964", + "diversity": { + "train": { + "mean_centered_corr": -0.017583154141902924, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05003640055656433 + }, + "val": { + "mean_centered_corr": -0.018760371953248978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07571490854024887 + }, + "test": { + "mean_centered_corr": -0.019624220207333565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09894119203090668 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 28, + 2, + 27, + 12, + 24, + 9, + 3, + 21, + 28 + ], + "head_selection_time": "0:00:00.166841" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..7dee0435b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,235 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.8027742505073547, + "min_features_per_member": 296.0, + "mean_features_per_member": 314.6875, + "max_features_per_member": 330.0, + "min_members_per_feature": 18.0, + "mean_members_per_feature": 25.68877601623535, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6710553952491272, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 592, + "metrics": { + "train": { + "rmse": 0.24172454855386824, + "mae": 0.1285606175661087, + "r2": 0.6267021894454956, + "score": -0.24172454855386824 + }, + "val": { + "rmse": 0.23377133352416926, + "mae": 0.1269364356994629, + "r2": 0.6349656581878662, + "score": -0.23377133352416926 + }, + "test": { + "rmse": 0.23643993065905872, + "mae": 0.152057945728302, + "r2": 0.6179842948913574, + "score": -0.23643993065905872 + } + }, + "time": "0:00:52.144209", + "diversity": { + "train": { + "mean_centered_corr": -0.01851801574230194, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785211756825447 + }, + "val": { + "mean_centered_corr": -0.019852403551340103, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0691390112042427 + }, + "test": { + "mean_centered_corr": -0.019887704402208328, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08872178196907043 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1, + 4, + 0, + 6, + 17, + 18, + 30, + 23, + 17, + 4 + ], + "head_selection_time": "0:00:00.159814" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..1ea10ba61 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,238 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.8008609414100647, + "min_features_per_member": 293.0, + "mean_features_per_member": 313.9375, + "max_features_per_member": 331.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.62755012512207, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6674885740537666, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1110, + "metrics": { + "train": { + "rmse": 0.21430787098953585, + "mae": 0.11510790884494781, + "r2": 0.7065796852111816, + "score": -0.21430787098953585 + }, + "val": { + "rmse": 0.23350988305065445, + "mae": 0.12613466382026672, + "r2": 0.6357816457748413, + "score": -0.23350988305065445 + }, + "test": { + "rmse": 0.23821200474658916, + "mae": 0.15480704605579376, + "r2": 0.6122365593910217, + "score": -0.23821200474658916 + } + }, + "time": "0:01:06.264634", + "diversity": { + "train": { + "mean_centered_corr": -0.01694483496248722, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0545165091753006 + }, + "val": { + "mean_centered_corr": -0.015533894300460815, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08842652291059494 + }, + "test": { + "mean_centered_corr": -0.016092214733362198, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11407508701086044 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 2, + 22, + 24, + 9, + 20, + 5, + 14, + 22, + 16, + 30, + 24, + 9, + 22 + ], + "head_selection_time": "0:00:00.203418" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..45688715d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22423689267064406, + "mae": 0.12748517096042633, + "r2": 0.6787611842155457, + "score": -0.22423689267064406 + }, + "val": { + "rmse": 0.2370483756746748, + "mae": 0.13395926356315613, + "r2": 0.6246596574783325, + "score": -0.2370483756746748 + }, + "test": { + "rmse": 0.23920040609688478, + "mae": 0.14310523867607117, + "r2": 0.6090120077133179, + "score": -0.23920040609688478 + } + }, + "time": "0:01:09.057982", + "diversity": { + "train": { + "mean_centered_corr": -0.019481390714645386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05119703337550163 + }, + "val": { + "mean_centered_corr": -0.01973012462258339, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07138136774301529 + }, + "test": { + "mean_centered_corr": -0.016405168920755386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09870413690805435 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015930" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/1/report.json new file mode 100644 index 000000000..e42d7cf5c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/1/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.227632937078759, + "mae": 0.123143769800663, + "r2": 0.6689572334289551, + "score": -0.227632937078759 + }, + "val": { + "rmse": 0.23898563084143007, + "mae": 0.13369078934192657, + "r2": 0.618499755859375, + "score": -0.23898563084143007 + }, + "test": { + "rmse": 0.23816985512587505, + "mae": 0.14563022553920746, + "r2": 0.6123737096786499, + "score": -0.23816985512587505 + } + }, + "time": "0:01:10.237258", + "diversity": { + "train": { + "mean_centered_corr": -0.018491873517632484, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04902740567922592 + }, + "val": { + "mean_centered_corr": -0.017897142097353935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07918199896812439 + }, + "test": { + "mean_centered_corr": -0.016073623672127724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1072448268532753 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1 + ], + "head_selection_time": "0:00:00.015321" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/2/report.json new file mode 100644 index 000000000..53d5ea3b7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-best-head-evaluation/2/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2373874975071392, + "mae": 0.13008663058280945, + "r2": 0.6399775743484497, + "score": -0.2373874975071392 + }, + "val": { + "rmse": 0.2381416834471136, + "mae": 0.13739991188049316, + "r2": 0.6211894750595093, + "score": -0.2381416834471136 + }, + "test": { + "rmse": 0.2498549025398126, + "mae": 0.1722332090139389, + "r2": 0.573405385017395, + "score": -0.2498549025398126 + } + }, + "time": "0:01:04.094967", + "diversity": { + "train": { + "mean_centered_corr": -0.019239937886595726, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04747721925377846 + }, + "val": { + "mean_centered_corr": -0.019888369366526604, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06191844120621681 + }, + "test": { + "mean_centered_corr": -0.019547125324606895, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07305453717708588 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015066" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml new file mode 100644 index 000000000..44f3d87b5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0/report.json new file mode 100644 index 000000000..475cad836 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22159317613989218, + "mae": 0.12286020070314407, + "r2": 0.6862912178039551, + "score": -0.22159317613989218 + }, + "val": { + "rmse": 0.2354682854519358, + "mae": 0.13157758116722107, + "r2": 0.6296467781066895, + "score": -0.2354682854519358 + }, + "test": { + "rmse": 0.24014251007009368, + "mae": 0.15741941332817078, + "r2": 0.6059260964393616, + "score": -0.24014251007009368 + } + }, + "time": "0:01:09.057982", + "diversity": { + "train": { + "mean_centered_corr": -0.019481390714645386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05119703337550163 + }, + "val": { + "mean_centered_corr": -0.01973012462258339, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07138136774301529 + }, + "test": { + "mean_centered_corr": -0.016405168920755386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09870413690805435 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml new file mode 100644 index 000000000..a9dd39142 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1/report.json new file mode 100644 index 000000000..cc70515b7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2219400179657564, + "mae": 0.11737057566642761, + "r2": 0.6853084564208984, + "score": -0.2219400179657564 + }, + "val": { + "rmse": 0.23689852205208367, + "mae": 0.12702235579490662, + "r2": 0.6251341104507446, + "score": -0.23689852205208367 + }, + "test": { + "rmse": 0.23132638249744272, + "mae": 0.13775070011615753, + "r2": 0.6343294978141785, + "score": -0.23132638249744272 + } + }, + "time": "0:01:10.237258", + "diversity": { + "train": { + "mean_centered_corr": -0.018491873517632484, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04902740567922592 + }, + "val": { + "mean_centered_corr": -0.017897142097353935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07918199896812439 + }, + "test": { + "mean_centered_corr": -0.016073623672127724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1072448268532753 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml new file mode 100644 index 000000000..05b5f4603 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2/report.json new file mode 100644 index 000000000..c8ac09e7e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2320526781045563, + "mae": 0.12807966768741608, + "r2": 0.6559773683547974, + "score": -0.2320526781045563 + }, + "val": { + "rmse": 0.2359145620703149, + "mae": 0.1308077573776245, + "r2": 0.6282416582107544, + "score": -0.2359145620703149 + }, + "test": { + "rmse": 0.2332252993496603, + "mae": 0.14604337513446808, + "r2": 0.6283013820648193, + "score": -0.2332252993496603 + } + }, + "time": "0:01:04.094967", + "diversity": { + "train": { + "mean_centered_corr": -0.019239937886595726, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04747721925377846 + }, + "val": { + "mean_centered_corr": -0.019888369366526604, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06191844120621681 + }, + "test": { + "mean_centered_corr": -0.019547125324606895, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07305453717708588 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..42d104b71 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,206 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22248795490129444, + "mae": 0.1261499673128128, + "r2": 0.6837526559829712, + "score": -0.22248795490129444 + }, + "val": { + "rmse": 0.2328613038082034, + "mae": 0.12715381383895874, + "r2": 0.6378021240234375, + "score": -0.2328613038082034 + }, + "test": { + "rmse": 0.23498425768312764, + "mae": 0.14776451885700226, + "r2": 0.6226736307144165, + "score": -0.23498425768312764 + } + }, + "time": "0:01:09.057982", + "diversity": { + "train": { + "mean_centered_corr": -0.019481390714645386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05119703337550163 + }, + "val": { + "mean_centered_corr": -0.01973012462258339, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07138136774301529 + }, + "test": { + "mean_centered_corr": -0.016405168920755386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09870413690805435 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 21, + 4, + 24 + ], + "head_selection_time": "0:00:00.076523" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..36bb022b8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1/report.json @@ -0,0 +1,206 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2235184256182525, + "mae": 0.12183725833892822, + "r2": 0.6808164119720459, + "score": -0.2235184256182525 + }, + "val": { + "rmse": 0.23460022912632159, + "mae": 0.12736009061336517, + "r2": 0.6323723793029785, + "score": -0.23460022912632159 + }, + "test": { + "rmse": 0.23076016380764466, + "mae": 0.13752052187919617, + "r2": 0.6361174583435059, + "score": -0.23076016380764466 + } + }, + "time": "0:01:10.237258", + "diversity": { + "train": { + "mean_centered_corr": -0.018491873517632484, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04902740567922592 + }, + "val": { + "mean_centered_corr": -0.017897142097353935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07918199896812439 + }, + "test": { + "mean_centered_corr": -0.016073623672127724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1072448268532753 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 1, + 19, + 28, + 6 + ], + "head_selection_time": "0:00:00.076399" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..234b01fa9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2/report.json @@ -0,0 +1,215 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2323423137248424, + "mae": 0.12613606452941895, + "r2": 0.6551179885864258, + "score": -0.2323423137248424 + }, + "val": { + "rmse": 0.23410085698681354, + "mae": 0.12800754606723785, + "r2": 0.633935809135437, + "score": -0.23410085698681354 + }, + "test": { + "rmse": 0.23135944978887066, + "mae": 0.14349235594272614, + "r2": 0.6342249512672424, + "score": -0.23135944978887066 + } + }, + "time": "0:01:04.094967", + "diversity": { + "train": { + "mean_centered_corr": -0.019239937886595726, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04747721925377846 + }, + "val": { + "mean_centered_corr": -0.019888369366526604, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06191844120621681 + }, + "test": { + "mean_centered_corr": -0.019547125324606895, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07305453717708588 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 30, + 5, + 12, + 13, + 30, + 12, + 15, + 9, + 11, + 22, + 2, + 23 + ], + "head_selection_time": "0:00:00.202277" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0.toml deleted file mode 100644 index 75c67868d..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0.toml +++ /dev/null @@ -1,35 +0,0 @@ -dataset = "homesite-insurance" -variant = "all_four_combined" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0/report.json deleted file mode 100644 index b1fd177fb..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0/report.json +++ /dev/null @@ -1,184 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 19, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_four_combined" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.9281404176737119, - "train_loss": 0.3755828700282357, - "val_score": 0.9251322733226822 - }, - { - "epoch": 1, - "test_score": 0.9402497021027693, - "train_loss": 0.25816836140372534, - "val_score": 0.936835678138968 - }, - { - "epoch": 2, - "test_score": 0.9400969237598616, - "train_loss": 0.25369841456413267, - "val_score": 0.9375367433223097 - }, - { - "epoch": 3, - "test_score": 0.9444803308575334, - "train_loss": 0.24889986542138187, - "val_score": 0.9411922560440782 - }, - { - "epoch": 4, - "test_score": 0.9431998850066914, - "train_loss": 0.24193649928678165, - "val_score": 0.9412456241453959 - }, - { - "epoch": 5, - "test_score": 0.9439822004894514, - "train_loss": 0.23912576938217336, - "val_score": 0.9424856456072856 - }, - { - "epoch": 6, - "test_score": 0.9431244381656779, - "train_loss": 0.23391161195256494, - "val_score": 0.9426291592016532 - }, - { - "epoch": 7, - "test_score": 0.9442989443289539, - "train_loss": 0.22139542956243863, - "val_score": 0.9440450404267323 - }, - { - "epoch": 8, - "test_score": 0.9454798172632811, - "train_loss": 0.21870070519772442, - "val_score": 0.9446249591408717 - }, - { - "epoch": 9, - "test_score": 0.9452781223055992, - "train_loss": 0.21202451532537286, - "val_score": 0.9450447383826217 - }, - { - "epoch": 10, - "test_score": 0.9455226932983104, - "train_loss": 0.21138894761150534, - "val_score": 0.9456571615235437 - }, - { - "epoch": 11, - "test_score": 0.9470071536894556, - "train_loss": 0.20695550563660534, - "val_score": 0.9463883748484702 - }, - { - "epoch": 12, - "test_score": 0.9496032498898924, - "train_loss": 0.20267397273670543, - "val_score": 0.9468419597491273 - }, - { - "epoch": 13, - "test_score": 0.9479806172776625, - "train_loss": 0.20230170339345932, - "val_score": 0.9473267214681163 - }, - { - "epoch": 14, - "test_score": 0.946599535368271, - "train_loss": 0.19895012825727462, - "val_score": 0.9474537674424232 - }, - { - "epoch": 15, - "test_score": 0.9487991279142535, - "train_loss": 0.19738293412056837, - "val_score": 0.9477246874875647 - }, - { - "epoch": 16, - "test_score": 0.9474314923353743, - "train_loss": 0.19684043594382025, - "val_score": 0.9485214284004848 - }, - { - "epoch": 17, - "test_score": 0.9470441220374943, - "train_loss": 0.1940444149754264, - "val_score": 0.9475463043891069 - }, - { - "epoch": 18, - "test_score": 0.9491013260377306, - "train_loss": 0.19452549476515163, - "val_score": 0.9486381436467773 - }, - { - "epoch": 19, - "test_score": 0.9488481290971439, - "train_loss": 0.19246512664989993, - "val_score": 0.948915692942066 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9488481290971439, - "score": 0.9488481290971439 - }, - "val": { - "roc-auc": 0.948915692942066, - "score": 0.948915692942066 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/0", - "seed": 0, - "task_type": "binclass", - "time_seconds": 93.77295994758606, - "variant": "all_four_combined" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1.toml deleted file mode 100644 index ca8babc0b..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1.toml +++ /dev/null @@ -1,35 +0,0 @@ -dataset = "homesite-insurance" -variant = "all_four_combined" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1/report.json deleted file mode 100644 index eef49da80..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1/report.json +++ /dev/null @@ -1,184 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 16, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_four_combined" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.9288299015269522, - "train_loss": 0.37169375148686495, - "val_score": 0.9242781990673085 - }, - { - "epoch": 1, - "test_score": 0.9391231943861461, - "train_loss": 0.26537797302007676, - "val_score": 0.9354944858794071 - }, - { - "epoch": 2, - "test_score": 0.9417845409289871, - "train_loss": 0.25233914811502806, - "val_score": 0.9386876479678965 - }, - { - "epoch": 3, - "test_score": 0.9425111743476582, - "train_loss": 0.24839151420376518, - "val_score": 0.9399943663682699 - }, - { - "epoch": 4, - "test_score": 0.9441030483278274, - "train_loss": 0.23347439156337219, - "val_score": 0.9413839416033751 - }, - { - "epoch": 5, - "test_score": 0.9455693990608975, - "train_loss": 0.23166601644320922, - "val_score": 0.9424608342758328 - }, - { - "epoch": 6, - "test_score": 0.9445222283186436, - "train_loss": 0.23105967830527913, - "val_score": 0.9430947277468074 - }, - { - "epoch": 7, - "test_score": 0.9461559755976042, - "train_loss": 0.2228196923028339, - "val_score": 0.9438730667756982 - }, - { - "epoch": 8, - "test_score": 0.9464112867411015, - "train_loss": 0.21721381694078445, - "val_score": 0.9451223990809644 - }, - { - "epoch": 9, - "test_score": 0.946084950460964, - "train_loss": 0.21338909674774517, - "val_score": 0.9455819538235977 - }, - { - "epoch": 10, - "test_score": 0.9466914729919881, - "train_loss": 0.21197187114845623, - "val_score": 0.9456509279182992 - }, - { - "epoch": 11, - "test_score": 0.9465127443184965, - "train_loss": 0.20536432604898106, - "val_score": 0.9458262601549519 - }, - { - "epoch": 12, - "test_score": 0.945089656217559, - "train_loss": 0.20558685606176202, - "val_score": 0.9459539479532121 - }, - { - "epoch": 13, - "test_score": 0.9471234106872256, - "train_loss": 0.20297221622683786, - "val_score": 0.9451995410448792 - }, - { - "epoch": 14, - "test_score": 0.9479524560948916, - "train_loss": 0.2004195828329433, - "val_score": 0.9473089526159311 - }, - { - "epoch": 15, - "test_score": 0.9499482515614414, - "train_loss": 0.19796687323938716, - "val_score": 0.9473182986277434 - }, - { - "epoch": 16, - "test_score": 0.9496361106437045, - "train_loss": 0.19757200046019122, - "val_score": 0.9483863024780541 - }, - { - "epoch": 17, - "test_score": 0.9472061183050925, - "train_loss": 0.19716880091212013, - "val_score": 0.9478469505546034 - }, - { - "epoch": 18, - "test_score": 0.9489066502337125, - "train_loss": 0.19436502781781284, - "val_score": 0.9483601987065858 - }, - { - "epoch": 19, - "test_score": 0.9500448404315621, - "train_loss": 0.19395294690674, - "val_score": 0.9482018792007486 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9496361106437045, - "score": 0.9496361106437045 - }, - "val": { - "roc-auc": 0.9483863024780541, - "score": 0.9483863024780541 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/1", - "seed": 1, - "task_type": "binclass", - "time_seconds": 96.05033469200134, - "variant": "all_four_combined" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2.toml deleted file mode 100644 index 1b0e6fde6..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2.toml +++ /dev/null @@ -1,35 +0,0 @@ -dataset = "homesite-insurance" -variant = "all_four_combined" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2/report.json deleted file mode 100644 index 5deeb3675..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2/report.json +++ /dev/null @@ -1,184 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 19, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_four_combined" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.9290865174356744, - "train_loss": 0.3692213460803032, - "val_score": 0.9257781416460449 - }, - { - "epoch": 1, - "test_score": 0.9399931707621637, - "train_loss": 0.26565207486802883, - "val_score": 0.9366023707359086 - }, - { - "epoch": 2, - "test_score": 0.9425727882610563, - "train_loss": 0.250053295628591, - "val_score": 0.9397225055674711 - }, - { - "epoch": 3, - "test_score": 0.9405813057684445, - "train_loss": 0.25021838085217907, - "val_score": 0.9381491225026873 - }, - { - "epoch": 4, - "test_score": 0.9423215363871476, - "train_loss": 0.2392362048680132, - "val_score": 0.9406268267254861 - }, - { - "epoch": 5, - "test_score": 0.9441924005834136, - "train_loss": 0.2352831008759412, - "val_score": 0.9422248276953261 - }, - { - "epoch": 6, - "test_score": 0.9442216490705383, - "train_loss": 0.2291740291497924, - "val_score": 0.9434997626218505 - }, - { - "epoch": 7, - "test_score": 0.9457917890447852, - "train_loss": 0.2204523051326925, - "val_score": 0.9440485748545299 - }, - { - "epoch": 8, - "test_score": 0.9475897313623712, - "train_loss": 0.21289140324700961, - "val_score": 0.9451809721107797 - }, - { - "epoch": 9, - "test_score": 0.9476547884062236, - "train_loss": 0.2199919426983053, - "val_score": 0.945304536409943 - }, - { - "epoch": 10, - "test_score": 0.9467265445979869, - "train_loss": 0.21338573572310535, - "val_score": 0.946054151618899 - }, - { - "epoch": 11, - "test_score": 0.9477642074677223, - "train_loss": 0.20716341110793027, - "val_score": 0.9463127451272946 - }, - { - "epoch": 12, - "test_score": 0.9476992953977839, - "train_loss": 0.20451430732553655, - "val_score": 0.9465820210480627 - }, - { - "epoch": 13, - "test_score": 0.9472819638243694, - "train_loss": 0.20352806760506195, - "val_score": 0.9467325507453713 - }, - { - "epoch": 14, - "test_score": 0.9486999415948425, - "train_loss": 0.20124661556699058, - "val_score": 0.9476719524180912 - }, - { - "epoch": 15, - "test_score": 0.947577468985489, - "train_loss": 0.19827090759168972, - "val_score": 0.9471636630155559 - }, - { - "epoch": 16, - "test_score": 0.9488903165060824, - "train_loss": 0.1969674747098576, - "val_score": 0.9474078726337114 - }, - { - "epoch": 17, - "test_score": 0.9474682794660205, - "train_loss": 0.19572840292345395, - "val_score": 0.947753525604916 - }, - { - "epoch": 18, - "test_score": 0.9480860374753701, - "train_loss": 0.19547649269754236, - "val_score": 0.9481652161064331 - }, - { - "epoch": 19, - "test_score": 0.9502789974647928, - "train_loss": 0.19355940371751784, - "val_score": 0.9485711389844801 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9502789974647928, - "score": 0.9502789974647928 - }, - "val": { - "roc-auc": 0.9485711389844801, - "score": 0.9485711389844801 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_four_combined-evaluation/2", - "seed": 2, - "task_type": "binclass", - "time_seconds": 96.36672067642212, - "variant": "all_four_combined" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml deleted file mode 100644 index 7c1e38fdb..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml +++ /dev/null @@ -1,34 +0,0 @@ -dataset = "homesite-insurance" -variant = "all_minus_cf_fisd" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0/report.json deleted file mode 100644 index 42f738bda..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0/report.json +++ /dev/null @@ -1,183 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 19, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_minus_cf_fisd" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.8867273748152669, - "train_loss": 0.4190130883997137, - "val_score": 0.8861749045533049 - }, - { - "epoch": 1, - "test_score": 0.9354356379127661, - "train_loss": 0.2752422061833468, - "val_score": 0.9309266599725895 - }, - { - "epoch": 2, - "test_score": 0.9386020494092362, - "train_loss": 0.25981232632290235, - "val_score": 0.9352112305054106 - }, - { - "epoch": 3, - "test_score": 0.9428584231148728, - "train_loss": 0.2525121568278833, - "val_score": 0.939197440821062 - }, - { - "epoch": 4, - "test_score": 0.9422175055227813, - "train_loss": 0.24629227424209768, - "val_score": 0.9399979447566122 - }, - { - "epoch": 5, - "test_score": 0.9429763352314928, - "train_loss": 0.24511424750089644, - "val_score": 0.9412687298077129 - }, - { - "epoch": 6, - "test_score": 0.9424501886545541, - "train_loss": 0.2420458352023905, - "val_score": 0.9414001102917317 - }, - { - "epoch": 7, - "test_score": 0.9441893923747006, - "train_loss": 0.22993632825938137, - "val_score": 0.9436276438465077 - }, - { - "epoch": 8, - "test_score": 0.945197794676152, - "train_loss": 0.2305623028766025, - "val_score": 0.9442499756942149 - }, - { - "epoch": 9, - "test_score": 0.9449498047153054, - "train_loss": 0.22382904548536647, - "val_score": 0.9449283748206871 - }, - { - "epoch": 10, - "test_score": 0.9442909466013718, - "train_loss": 0.22422936938025734, - "val_score": 0.9454736702097868 - }, - { - "epoch": 11, - "test_score": 0.9444200700339982, - "train_loss": 0.21886501434174452, - "val_score": 0.9449596131837814 - }, - { - "epoch": 12, - "test_score": 0.9465477313563789, - "train_loss": 0.21386859565973282, - "val_score": 0.9465164934600688 - }, - { - "epoch": 13, - "test_score": 0.9456236676293256, - "train_loss": 0.21183434413238006, - "val_score": 0.9469095183142883 - }, - { - "epoch": 14, - "test_score": 0.9448259728305354, - "train_loss": 0.2103140194307674, - "val_score": 0.9473692928596451 - }, - { - "epoch": 15, - "test_score": 0.9467284896646648, - "train_loss": 0.20671901418404146, - "val_score": 0.9475178003718955 - }, - { - "epoch": 16, - "test_score": 0.9442787687926061, - "train_loss": 0.20530494559894907, - "val_score": 0.9480879774293199 - }, - { - "epoch": 17, - "test_score": 0.9446361174091539, - "train_loss": 0.1996095741336996, - "val_score": 0.9480463995461023 - }, - { - "epoch": 18, - "test_score": 0.944189694403688, - "train_loss": 0.2015354111790657, - "val_score": 0.948330235199289 - }, - { - "epoch": 19, - "test_score": 0.9438768769410195, - "train_loss": 0.19719995788552544, - "val_score": 0.9486877399333562 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9438768769410195, - "score": 0.9438768769410195 - }, - "val": { - "roc-auc": 0.9486877399333562, - "score": 0.9486877399333562 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/0", - "seed": 0, - "task_type": "binclass", - "time_seconds": 94.5395917892456, - "variant": "all_minus_cf_fisd" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1.toml deleted file mode 100644 index 17983d828..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1.toml +++ /dev/null @@ -1,34 +0,0 @@ -dataset = "homesite-insurance" -variant = "all_minus_cf_fisd" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1/report.json deleted file mode 100644 index 9ec7484a7..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1/report.json +++ /dev/null @@ -1,183 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 19, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_minus_cf_fisd" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.9011766348635782, - "train_loss": 0.4144984155893326, - "val_score": 0.8989568989190753 - }, - { - "epoch": 1, - "test_score": 0.9314295979129168, - "train_loss": 0.2808895646171136, - "val_score": 0.925805326846914 - }, - { - "epoch": 2, - "test_score": 0.9368095194511018, - "train_loss": 0.2609179992567409, - "val_score": 0.9320041505084555 - }, - { - "epoch": 3, - "test_score": 0.9408773425005816, - "train_loss": 0.25344904268329793, - "val_score": 0.9373312013993098 - }, - { - "epoch": 4, - "test_score": 0.9423485377786072, - "train_loss": 0.23822286413474517, - "val_score": 0.9391652441180923 - }, - { - "epoch": 5, - "test_score": 0.9437661410331363, - "train_loss": 0.2377325576814738, - "val_score": 0.9408794855603414 - }, - { - "epoch": 6, - "test_score": 0.9432573067577464, - "train_loss": 0.2404585752974857, - "val_score": 0.9416476697033968 - }, - { - "epoch": 7, - "test_score": 0.9425979653974329, - "train_loss": 0.23373751518401234, - "val_score": 0.9408246315926113 - }, - { - "epoch": 8, - "test_score": 0.9462087340210961, - "train_loss": 0.22808001962575045, - "val_score": 0.9446144877391139 - }, - { - "epoch": 9, - "test_score": 0.9455337717215624, - "train_loss": 0.22486585839228196, - "val_score": 0.9451965341436188 - }, - { - "epoch": 10, - "test_score": 0.9457187463545103, - "train_loss": 0.22854889671910894, - "val_score": 0.9444836875342859 - }, - { - "epoch": 11, - "test_score": 0.9457179489979838, - "train_loss": 0.21878958547657187, - "val_score": 0.9453681209418593 - }, - { - "epoch": 12, - "test_score": 0.9440899885944188, - "train_loss": 0.21896708648313176, - "val_score": 0.9451183810871748 - }, - { - "epoch": 13, - "test_score": 0.9454167173672661, - "train_loss": 0.21345828541300513, - "val_score": 0.9444320778547578 - }, - { - "epoch": 14, - "test_score": 0.9462222528385651, - "train_loss": 0.213582714443857, - "val_score": 0.9465140316695633 - }, - { - "epoch": 15, - "test_score": 0.9449545163675065, - "train_loss": 0.2090706546198238, - "val_score": 0.9466988769680983 - }, - { - "epoch": 16, - "test_score": 0.9479850993878329, - "train_loss": 0.20644646869464353, - "val_score": 0.9477843859073254 - }, - { - "epoch": 17, - "test_score": 0.943974396060421, - "train_loss": 0.20588010414080185, - "val_score": 0.9470860462777578 - }, - { - "epoch": 18, - "test_score": 0.9452268136212463, - "train_loss": 0.19995117322965103, - "val_score": 0.9480954155534905 - }, - { - "epoch": 19, - "test_score": 0.9468154377695126, - "train_loss": 0.20072065158323807, - "val_score": 0.9483137412029014 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9468154377695126, - "score": 0.9468154377695126 - }, - "val": { - "roc-auc": 0.9483137412029014, - "score": 0.9483137412029014 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/1", - "seed": 1, - "task_type": "binclass", - "time_seconds": 96.52392911911011, - "variant": "all_minus_cf_fisd" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2.toml deleted file mode 100644 index a1d5dd1d2..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2.toml +++ /dev/null @@ -1,34 +0,0 @@ -dataset = "homesite-insurance" -variant = "all_minus_cf_fisd" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2/report.json deleted file mode 100644 index e6cf07544..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2/report.json +++ /dev/null @@ -1,183 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 19, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_minus_cf_fisd" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.9040055712508612, - "train_loss": 0.4124534674666145, - "val_score": 0.9022017146476378 - }, - { - "epoch": 1, - "test_score": 0.9323568148225967, - "train_loss": 0.2793294776569713, - "val_score": 0.9278128027068161 - }, - { - "epoch": 2, - "test_score": 0.938795239230637, - "train_loss": 0.25687101740728724, - "val_score": 0.9354104772784034 - }, - { - "epoch": 3, - "test_score": 0.9392835959406918, - "train_loss": 0.2551576859571717, - "val_score": 0.9361047197852009 - }, - { - "epoch": 4, - "test_score": 0.9422998627870229, - "train_loss": 0.2425997487523339, - "val_score": 0.9403174060352589 - }, - { - "epoch": 5, - "test_score": 0.9433914559527208, - "train_loss": 0.24269262145866047, - "val_score": 0.9411395913114763 - }, - { - "epoch": 6, - "test_score": 0.9437721816128812, - "train_loss": 0.23879283395680515, - "val_score": 0.9426016223164263 - }, - { - "epoch": 7, - "test_score": 0.9451093968321654, - "train_loss": 0.22980331656607714, - "val_score": 0.943478301083907 - }, - { - "epoch": 8, - "test_score": 0.9471574433135084, - "train_loss": 0.22093958367000927, - "val_score": 0.9447153420208632 - }, - { - "epoch": 9, - "test_score": 0.9459051344831182, - "train_loss": 0.235467699711973, - "val_score": 0.9443288233272656 - }, - { - "epoch": 10, - "test_score": 0.9456907663891317, - "train_loss": 0.2297248977151784, - "val_score": 0.945183521822375 - }, - { - "epoch": 11, - "test_score": 0.9477198575312351, - "train_loss": 0.21863165281035685, - "val_score": 0.9454252344815891 - }, - { - "epoch": 12, - "test_score": 0.9457698617403113, - "train_loss": 0.21798922636292198, - "val_score": 0.9455848640116595 - }, - { - "epoch": 13, - "test_score": 0.946250172398146, - "train_loss": 0.21708055368878626, - "val_score": 0.9462267583017775 - }, - { - "epoch": 14, - "test_score": 0.9465536873680073, - "train_loss": 0.21340600766918877, - "val_score": 0.9464460686673908 - }, - { - "epoch": 15, - "test_score": 0.9450038799851819, - "train_loss": 0.2088709598237818, - "val_score": 0.9465693604111766 - }, - { - "epoch": 16, - "test_score": 0.9451183610525068, - "train_loss": 0.20932438238100573, - "val_score": 0.9471085189082304 - }, - { - "epoch": 17, - "test_score": 0.9446490321686482, - "train_loss": 0.2056827260689302, - "val_score": 0.9475481507319862 - }, - { - "epoch": 18, - "test_score": 0.9433471664220312, - "train_loss": 0.20389442484487186, - "val_score": 0.9475102743266357 - }, - { - "epoch": 19, - "test_score": 0.947036245121507, - "train_loss": 0.2016631698066538, - "val_score": 0.9483005706236965 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.947036245121507, - "score": 0.947036245121507 - }, - "val": { - "roc-auc": 0.9483005706236965, - "score": 0.9483005706236965 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_cf_fisd-evaluation/2", - "seed": 2, - "task_type": "binclass", - "time_seconds": 95.87906002998352, - "variant": "all_minus_cf_fisd" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0.toml deleted file mode 100644 index c1324daa6..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "homesite-insurance" -variant = "all_minus_esam" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0/report.json deleted file mode 100644 index 4f0fb2a36..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0/report.json +++ /dev/null @@ -1,182 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 19, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_minus_esam" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.9282623648976015, - "train_loss": 0.37420019615780226, - "val_score": 0.9251574715069288 - }, - { - "epoch": 1, - "test_score": 0.9371021734585819, - "train_loss": 0.27270237613808024, - "val_score": 0.9346151255186971 - }, - { - "epoch": 2, - "test_score": 0.9409601467677244, - "train_loss": 0.254967031695626, - "val_score": 0.938594398860388 - }, - { - "epoch": 3, - "test_score": 0.9424858401562084, - "train_loss": 0.24498489864847878, - "val_score": 0.9402089202029422 - }, - { - "epoch": 4, - "test_score": 0.9423627935468052, - "train_loss": 0.23897522552446884, - "val_score": 0.9400627513916721 - }, - { - "epoch": 5, - "test_score": 0.9436719925572326, - "train_loss": 0.23254627517678522, - "val_score": 0.942443874297671 - }, - { - "epoch": 6, - "test_score": 0.9456386603482525, - "train_loss": 0.22714561522006987, - "val_score": 0.943583815183399 - }, - { - "epoch": 7, - "test_score": 0.9447918314738174, - "train_loss": 0.22587331425059926, - "val_score": 0.9446441698989243 - }, - { - "epoch": 8, - "test_score": 0.9470115995561479, - "train_loss": 0.21659111380577087, - "val_score": 0.9450748073952258 - }, - { - "epoch": 9, - "test_score": 0.9469706444254775, - "train_loss": 0.2131239863959226, - "val_score": 0.945244732484875 - }, - { - "epoch": 10, - "test_score": 0.9454978181909208, - "train_loss": 0.21075186621059072, - "val_score": 0.9459386496836415 - }, - { - "epoch": 11, - "test_score": 0.9478734332306693, - "train_loss": 0.2058184477415952, - "val_score": 0.9464268579093381 - }, - { - "epoch": 12, - "test_score": 0.9481618225888494, - "train_loss": 0.2057848500934514, - "val_score": 0.9464468423729783 - }, - { - "epoch": 13, - "test_score": 0.9474614777732278, - "train_loss": 0.19984769130294974, - "val_score": 0.9469739644728812 - }, - { - "epoch": 14, - "test_score": 0.9473789151292749, - "train_loss": 0.20040455880490216, - "val_score": 0.9474729166557132 - }, - { - "epoch": 15, - "test_score": 0.9472418060502253, - "train_loss": 0.19612910435958342, - "val_score": 0.9469740523939708 - }, - { - "epoch": 16, - "test_score": 0.946604150371196, - "train_loss": 0.19511878978122366, - "val_score": 0.9478675768421971 - }, - { - "epoch": 17, - "test_score": 0.945813837160854, - "train_loss": 0.1943703759800304, - "val_score": 0.9468067737291153 - }, - { - "epoch": 18, - "test_score": 0.9494150737496799, - "train_loss": 0.1947024403647943, - "val_score": 0.9476518712412524 - }, - { - "epoch": 19, - "test_score": 0.9485440704751056, - "train_loss": 0.19200101332231, - "val_score": 0.9489494986009732 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9485440704751056, - "score": 0.9485440704751056 - }, - "val": { - "roc-auc": 0.9489494986009732, - "score": 0.9489494986009732 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/0", - "seed": 0, - "task_type": "binclass", - "time_seconds": 57.65527606010437, - "variant": "all_minus_esam" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1.toml deleted file mode 100644 index cd4d3cafc..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "homesite-insurance" -variant = "all_minus_esam" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1/report.json deleted file mode 100644 index 666a0d0d6..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1/report.json +++ /dev/null @@ -1,182 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 19, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_minus_esam" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.9311408582011115, - "train_loss": 0.36883535276759755, - "val_score": 0.9270232273626889 - }, - { - "epoch": 1, - "test_score": 0.9384909269042493, - "train_loss": 0.2649405887181109, - "val_score": 0.9355450668821883 - }, - { - "epoch": 2, - "test_score": 0.9405701306959163, - "train_loss": 0.2457744756882841, - "val_score": 0.9378696829039744 - }, - { - "epoch": 3, - "test_score": 0.941262356972361, - "train_loss": 0.2503055283969099, - "val_score": 0.9383217731461104 - }, - { - "epoch": 4, - "test_score": 0.9439972536141755, - "train_loss": 0.23776601932265543, - "val_score": 0.9409290378863756 - }, - { - "epoch": 5, - "test_score": 0.9465358555766004, - "train_loss": 0.23235300061377612, - "val_score": 0.9434845962339142 - }, - { - "epoch": 6, - "test_score": 0.945467603211037, - "train_loss": 0.22437328885902058, - "val_score": 0.943037869178237 - }, - { - "epoch": 7, - "test_score": 0.9459721486748079, - "train_loss": 0.22132552550597626, - "val_score": 0.9439452763664928 - }, - { - "epoch": 8, - "test_score": 0.9467611450387656, - "train_loss": 0.2172819274392995, - "val_score": 0.9445786950635843 - }, - { - "epoch": 9, - "test_score": 0.9446675042615081, - "train_loss": 0.21471701684323224, - "val_score": 0.9449388022619001 - }, - { - "epoch": 10, - "test_score": 0.9474626617268577, - "train_loss": 0.2101835078813813, - "val_score": 0.945552992616721 - }, - { - "epoch": 11, - "test_score": 0.9461134015915622, - "train_loss": 0.20978831648826599, - "val_score": 0.9451966924015798 - }, - { - "epoch": 12, - "test_score": 0.9470384922171722, - "train_loss": 0.2077528560703451, - "val_score": 0.9460920192321405 - }, - { - "epoch": 13, - "test_score": 0.9475360910142365, - "train_loss": 0.2031910687685013, - "val_score": 0.9464087285806861 - }, - { - "epoch": 14, - "test_score": 0.9476712912700865, - "train_loss": 0.20127531723542647, - "val_score": 0.9469623237206334 - }, - { - "epoch": 15, - "test_score": 0.9477483811487906, - "train_loss": 0.1988604185256091, - "val_score": 0.9468381615580617 - }, - { - "epoch": 16, - "test_score": 0.9478177149231024, - "train_loss": 0.1974608377976851, - "val_score": 0.9479933391685973 - }, - { - "epoch": 17, - "test_score": 0.9474401786890472, - "train_loss": 0.196479739790613, - "val_score": 0.9479001955663963 - }, - { - "epoch": 18, - "test_score": 0.9478719472480521, - "train_loss": 0.19606405632062393, - "val_score": 0.947817444236972 - }, - { - "epoch": 19, - "test_score": 0.9512055136672225, - "train_loss": 0.19362684867598795, - "val_score": 0.9482717237142363 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9512055136672225, - "score": 0.9512055136672225 - }, - "val": { - "roc-auc": 0.9482717237142363, - "score": 0.9482717237142363 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/1", - "seed": 1, - "task_type": "binclass", - "time_seconds": 60.01497411727905, - "variant": "all_minus_esam" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2.toml deleted file mode 100644 index fd0766099..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "homesite-insurance" -variant = "all_minus_esam" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2/report.json deleted file mode 100644 index 8ec991525..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2/report.json +++ /dev/null @@ -1,182 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 18, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_minus_esam" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.9305516117281577, - "train_loss": 0.36503461572256957, - "val_score": 0.9267991868424613 - }, - { - "epoch": 1, - "test_score": 0.9385174812928078, - "train_loss": 0.2688400410793044, - "val_score": 0.9351827001118724 - }, - { - "epoch": 2, - "test_score": 0.9415470978203752, - "train_loss": 0.2523974738337777, - "val_score": 0.9381120285950332 - }, - { - "epoch": 3, - "test_score": 0.9416739379138582, - "train_loss": 0.24787599999796261, - "val_score": 0.939197282563101 - }, - { - "epoch": 4, - "test_score": 0.9444743627647454, - "train_loss": 0.23736426600001076, - "val_score": 0.9420454422964482 - }, - { - "epoch": 5, - "test_score": 0.9440574902753915, - "train_loss": 0.23959832096641714, - "val_score": 0.9421820365010734 - }, - { - "epoch": 6, - "test_score": 0.9437026182965391, - "train_loss": 0.23264405889944598, - "val_score": 0.942717256133318 - }, - { - "epoch": 7, - "test_score": 0.9459018363265774, - "train_loss": 0.2235704476183111, - "val_score": 0.943923375223102 - }, - { - "epoch": 8, - "test_score": 0.9460808911913753, - "train_loss": 0.22024327313358133, - "val_score": 0.9448243114191718 - }, - { - "epoch": 9, - "test_score": 0.9465984118204385, - "train_loss": 0.2157839154655283, - "val_score": 0.9454886519634352 - }, - { - "epoch": 10, - "test_score": 0.9481819014759215, - "train_loss": 0.21046820960261606, - "val_score": 0.9453150341880276 - }, - { - "epoch": 11, - "test_score": 0.9455237564403451, - "train_loss": 0.21228812594305385, - "val_score": 0.9457623591071136 - }, - { - "epoch": 12, - "test_score": 0.947202747661595, - "train_loss": 0.2047921211882071, - "val_score": 0.9457797674828315 - }, - { - "epoch": 13, - "test_score": 0.9466076055828101, - "train_loss": 0.2042220937934789, - "val_score": 0.9464812986479477 - }, - { - "epoch": 14, - "test_score": 0.9488007467896252, - "train_loss": 0.199715626781637, - "val_score": 0.9472975668348425 - }, - { - "epoch": 15, - "test_score": 0.9474458326716885, - "train_loss": 0.19854097799821332, - "val_score": 0.9470691302601407 - }, - { - "epoch": 16, - "test_score": 0.9493707842189902, - "train_loss": 0.19561608921397816, - "val_score": 0.9477912789207413 - }, - { - "epoch": 17, - "test_score": 0.9485832617564904, - "train_loss": 0.19515144513411956, - "val_score": 0.9479748933240233 - }, - { - "epoch": 18, - "test_score": 0.9485803381158939, - "train_loss": 0.19575764618136665, - "val_score": 0.9482433076181145 - }, - { - "epoch": 19, - "test_score": 0.9506953021396505, - "train_loss": 0.19345392286777496, - "val_score": 0.9479477696679169 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9485803381158939, - "score": 0.9485803381158939 - }, - "val": { - "roc-auc": 0.9482433076181145, - "score": 0.9482433076181145 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_esam-evaluation/2", - "seed": 2, - "task_type": "binclass", - "time_seconds": 60.87337589263916, - "variant": "all_minus_esam" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0.toml deleted file mode 100644 index 84687163b..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "homesite-insurance" -variant = "all_minus_mfb" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0/report.json deleted file mode 100644 index eb71a0c05..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0/report.json +++ /dev/null @@ -1,182 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 16, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_minus_mfb" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.936080856397636, - "train_loss": 0.3485633905638348, - "val_score": 0.9330192082786779 - }, - { - "epoch": 1, - "test_score": 0.9443432701031219, - "train_loss": 0.21431942704049023, - "val_score": 0.9408591757886702 - }, - { - "epoch": 2, - "test_score": 0.9482861860446371, - "train_loss": 0.20039536370472474, - "val_score": 0.9447542646871785 - }, - { - "epoch": 3, - "test_score": 0.9473031058534765, - "train_loss": 0.1941048334945332, - "val_score": 0.9441183578232547 - }, - { - "epoch": 4, - "test_score": 0.9496835171135424, - "train_loss": 0.1905539946122603, - "val_score": 0.9447491564718793 - }, - { - "epoch": 5, - "test_score": 0.9511852052381201, - "train_loss": 0.18824819949540225, - "val_score": 0.9475763470253842 - }, - { - "epoch": 6, - "test_score": 0.9501237545653494, - "train_loss": 0.18603328303857283, - "val_score": 0.947620676838703 - }, - { - "epoch": 7, - "test_score": 0.9511574668959317, - "train_loss": 0.18426710014993494, - "val_score": 0.9483009486843812 - }, - { - "epoch": 8, - "test_score": 0.9534666597208054, - "train_loss": 0.18335880569436333, - "val_score": 0.9493766016694772 - }, - { - "epoch": 9, - "test_score": 0.9512971975865901, - "train_loss": 0.18194586864926598, - "val_score": 0.9486100880271222 - }, - { - "epoch": 10, - "test_score": 0.9522303946702174, - "train_loss": 0.18081731620160016, - "val_score": 0.9494002876109848 - }, - { - "epoch": 11, - "test_score": 0.9537785469341931, - "train_loss": 0.18012406162240288, - "val_score": 0.9505937636481007 - }, - { - "epoch": 12, - "test_score": 0.9533768967057965, - "train_loss": 0.179088133438067, - "val_score": 0.9504195392171756 - }, - { - "epoch": 13, - "test_score": 0.9528421120998224, - "train_loss": 0.17811415845697576, - "val_score": 0.950407643493768 - }, - { - "epoch": 14, - "test_score": 0.9529662460135799, - "train_loss": 0.1776683739640496, - "val_score": 0.9504370443060922 - }, - { - "epoch": 15, - "test_score": 0.9527797249922172, - "train_loss": 0.17663268406282773, - "val_score": 0.9503563942907068 - }, - { - "epoch": 16, - "test_score": 0.9532358612499129, - "train_loss": 0.17589410096406938, - "val_score": 0.951222742330173 - }, - { - "epoch": 17, - "test_score": 0.9526458536639111, - "train_loss": 0.17520266324281691, - "val_score": 0.9507549757577818 - }, - { - "epoch": 18, - "test_score": 0.9533179285663269, - "train_loss": 0.17484911680221557, - "val_score": 0.9511791246776792 - }, - { - "epoch": 19, - "test_score": 0.9527198990904239, - "train_loss": 0.17424605746160854, - "val_score": 0.9510088391115631 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9532358612499129, - "score": 0.9532358612499129 - }, - "val": { - "roc-auc": 0.951222742330173, - "score": 0.951222742330173 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/0", - "seed": 0, - "task_type": "binclass", - "time_seconds": 93.82961058616638, - "variant": "all_minus_mfb" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1.toml deleted file mode 100644 index 327c2f2a0..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "homesite-insurance" -variant = "all_minus_mfb" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1/report.json deleted file mode 100644 index e4b0b5126..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1/report.json +++ /dev/null @@ -1,182 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 16, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_minus_mfb" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.9349781123217293, - "train_loss": 0.3451001604849642, - "val_score": 0.9310866675633446 - }, - { - "epoch": 1, - "test_score": 0.9438002461463759, - "train_loss": 0.21620039181275802, - "val_score": 0.9403393247628676 - }, - { - "epoch": 2, - "test_score": 0.9468972997062153, - "train_loss": 0.20234638845378702, - "val_score": 0.9439125081764416 - }, - { - "epoch": 3, - "test_score": 0.9498045461693108, - "train_loss": 0.19468857971104708, - "val_score": 0.9458391669708881 - }, - { - "epoch": 4, - "test_score": 0.951420727442373, - "train_loss": 0.1909984537146308, - "val_score": 0.9467180877261508 - }, - { - "epoch": 5, - "test_score": 0.9516745526032531, - "train_loss": 0.18896782167933204, - "val_score": 0.9476140300043379 - }, - { - "epoch": 6, - "test_score": 0.9515406450314684, - "train_loss": 0.18661685626615176, - "val_score": 0.9481888580873961 - }, - { - "epoch": 7, - "test_score": 0.9531164269071972, - "train_loss": 0.18516999808224766, - "val_score": 0.9489969935735135 - }, - { - "epoch": 8, - "test_score": 0.9522627600964907, - "train_loss": 0.18365479057485407, - "val_score": 0.9487103796138988 - }, - { - "epoch": 9, - "test_score": 0.9513448577607774, - "train_loss": 0.1828564101999456, - "val_score": 0.9483731846515029 - }, - { - "epoch": 10, - "test_score": 0.9536176258897894, - "train_loss": 0.18148549510674042, - "val_score": 0.9493563534425687 - }, - { - "epoch": 11, - "test_score": 0.9524729722716125, - "train_loss": 0.1805111982605674, - "val_score": 0.949499594481559 - }, - { - "epoch": 12, - "test_score": 0.9531607647625244, - "train_loss": 0.17996388402852145, - "val_score": 0.9504649240835682 - }, - { - "epoch": 13, - "test_score": 0.9535825301214718, - "train_loss": 0.1787733332677321, - "val_score": 0.9495215132091678 - }, - { - "epoch": 14, - "test_score": 0.9532390990006563, - "train_loss": 0.1780598908662796, - "val_score": 0.9504280060180932 - }, - { - "epoch": 15, - "test_score": 0.9536520330320163, - "train_loss": 0.17685881514440882, - "val_score": 0.9504281554839451 - }, - { - "epoch": 16, - "test_score": 0.9528538308245276, - "train_loss": 0.17636417136950927, - "val_score": 0.9509531674777006 - }, - { - "epoch": 17, - "test_score": 0.9524469373729123, - "train_loss": 0.17591779868711124, - "val_score": 0.9503000896250001 - }, - { - "epoch": 18, - "test_score": 0.952751672539882, - "train_loss": 0.17490007105198774, - "val_score": 0.9499999797781494 - }, - { - "epoch": 19, - "test_score": 0.9532202282295332, - "train_loss": 0.17446433955972845, - "val_score": 0.9509216917276646 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9528538308245276, - "score": 0.9528538308245276 - }, - "val": { - "roc-auc": 0.9509531674777006, - "score": 0.9509531674777006 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/1", - "seed": 1, - "task_type": "binclass", - "time_seconds": 94.35714793205261, - "variant": "all_minus_mfb" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2.toml deleted file mode 100644 index cb4193e6e..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "homesite-insurance" -variant = "all_minus_mfb" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2/report.json deleted file mode 100644 index c8b69b870..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2/report.json +++ /dev/null @@ -1,182 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 19, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_minus_mfb" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.9348914662458687, - "train_loss": 0.34214188944209706, - "val_score": 0.9314159760040135 - }, - { - "epoch": 1, - "test_score": 0.9444708350661741, - "train_loss": 0.2145986951210282, - "val_score": 0.9404227355004626 - }, - { - "epoch": 2, - "test_score": 0.9474591823529247, - "train_loss": 0.20296300189061597, - "val_score": 0.9442555410991793 - }, - { - "epoch": 3, - "test_score": 0.9483235893144174, - "train_loss": 0.1946356772021814, - "val_score": 0.9448229574343938 - }, - { - "epoch": 4, - "test_score": 0.9502820177546653, - "train_loss": 0.1902028499679132, - "val_score": 0.9466603587387945 - }, - { - "epoch": 5, - "test_score": 0.9518550934506681, - "train_loss": 0.1883919584480199, - "val_score": 0.946771587709103 - }, - { - "epoch": 6, - "test_score": 0.948750694545859, - "train_loss": 0.1861147192391482, - "val_score": 0.9466200732955922 - }, - { - "epoch": 7, - "test_score": 0.9516568537046007, - "train_loss": 0.18476087220690468, - "val_score": 0.9477665027577242 - }, - { - "epoch": 8, - "test_score": 0.9528440571665004, - "train_loss": 0.18319841162724929, - "val_score": 0.9497558229126473 - }, - { - "epoch": 9, - "test_score": 0.9525012663471377, - "train_loss": 0.18201071254231713, - "val_score": 0.949482827929794 - }, - { - "epoch": 10, - "test_score": 0.9528152315199576, - "train_loss": 0.18100855905901303, - "val_score": 0.9496849497224129 - }, - { - "epoch": 11, - "test_score": 0.9535917480461624, - "train_loss": 0.17989628274332392, - "val_score": 0.9499481151274618 - }, - { - "epoch": 12, - "test_score": 0.9539670855091903, - "train_loss": 0.17919402501799844, - "val_score": 0.9502128543200125 - }, - { - "epoch": 13, - "test_score": 0.9540179955152801, - "train_loss": 0.17842414839701218, - "val_score": 0.9505214397598896 - }, - { - "epoch": 14, - "test_score": 0.9538338182388588, - "train_loss": 0.17782833643934942, - "val_score": 0.9511898422584875 - }, - { - "epoch": 15, - "test_score": 0.952540469709682, - "train_loss": 0.17649481784213672, - "val_score": 0.9504172268925222 - }, - { - "epoch": 16, - "test_score": 0.9527322701977416, - "train_loss": 0.1761845365166664, - "val_score": 0.9497746204415795 - }, - { - "epoch": 17, - "test_score": 0.952729129096274, - "train_loss": 0.17521973590959203, - "val_score": 0.9507788111651413 - }, - { - "epoch": 18, - "test_score": 0.952827276435969, - "train_loss": 0.1749107297171246, - "val_score": 0.9509241359339523 - }, - { - "epoch": 19, - "test_score": 0.953869240198483, - "train_loss": 0.17429973889480938, - "val_score": 0.9512909251350691 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.953869240198483, - "score": 0.953869240198483 - }, - "val": { - "roc-auc": 0.9512909251350691, - "score": 0.9512909251350691 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_mfb-evaluation/2", - "seed": 2, - "task_type": "binclass", - "time_seconds": 96.05296754837036, - "variant": "all_minus_mfb" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0.toml deleted file mode 100644 index bba72ccae..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0.toml +++ /dev/null @@ -1,32 +0,0 @@ -dataset = "homesite-insurance" -variant = "all_minus_rla" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0/report.json deleted file mode 100644 index 9c2d3f6fe..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0/report.json +++ /dev/null @@ -1,181 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 17, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_minus_rla" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.9290964481487749, - "train_loss": 0.3734414360739968, - "val_score": 0.9260200916922008 - }, - { - "epoch": 1, - "test_score": 0.9399295755386096, - "train_loss": 0.2582675423134457, - "val_score": 0.936778230499098 - }, - { - "epoch": 2, - "test_score": 0.939959705950377, - "train_loss": 0.2561210263859142, - "val_score": 0.9372083404688625 - }, - { - "epoch": 3, - "test_score": 0.9446191917047086, - "train_loss": 0.2515790993517095, - "val_score": 0.9414589031242706 - }, - { - "epoch": 4, - "test_score": 0.9436923009863349, - "train_loss": 0.2490062885663726, - "val_score": 0.9410485314390957 - }, - { - "epoch": 5, - "test_score": 0.9448191590565831, - "train_loss": 0.24894793981855565, - "val_score": 0.9423545728470804 - }, - { - "epoch": 6, - "test_score": 0.9446520041338828, - "train_loss": 0.24585373239083724, - "val_score": 0.9425746129577363 - }, - { - "epoch": 7, - "test_score": 0.9464069737671635, - "train_loss": 0.233190402794968, - "val_score": 0.9439372755473496 - }, - { - "epoch": 8, - "test_score": 0.9476272312814273, - "train_loss": 0.23624674583023245, - "val_score": 0.9448485776398697 - }, - { - "epoch": 9, - "test_score": 0.9479388889527848, - "train_loss": 0.23320516537536273, - "val_score": 0.945495896661209 - }, - { - "epoch": 10, - "test_score": 0.948785524528668, - "train_loss": 0.23475631827657872, - "val_score": 0.9462668503185829 - }, - { - "epoch": 11, - "test_score": 0.9475035081270926, - "train_loss": 0.23386801467700438, - "val_score": 0.9448978925789625 - }, - { - "epoch": 12, - "test_score": 0.9484173149497389, - "train_loss": 0.22608610892837697, - "val_score": 0.945860417498217 - }, - { - "epoch": 13, - "test_score": 0.9491997150006151, - "train_loss": 0.225085971030322, - "val_score": 0.9468609331202384 - }, - { - "epoch": 14, - "test_score": 0.948916822570003, - "train_loss": 0.23035850606181404, - "val_score": 0.946784046127483 - }, - { - "epoch": 15, - "test_score": 0.9502937606416895, - "train_loss": 0.2231764554977417, - "val_score": 0.9476667299053756 - }, - { - "epoch": 16, - "test_score": 0.949525338492343, - "train_loss": 0.22973006882450797, - "val_score": 0.9472551185328391 - }, - { - "epoch": 17, - "test_score": 0.950880361340715, - "train_loss": 0.21746040094982494, - "val_score": 0.9482617446705797 - }, - { - "epoch": 18, - "test_score": 0.9492371424327144, - "train_loss": 0.22736059007319537, - "val_score": 0.9470035675037112 - }, - { - "epoch": 19, - "test_score": 0.9503750910073745, - "train_loss": 0.21671231402592225, - "val_score": 0.9481422335336418 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.950880361340715, - "score": 0.950880361340715 - }, - "val": { - "roc-auc": 0.9482617446705797, - "score": 0.9482617446705797 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/0", - "seed": 0, - "task_type": "binclass", - "time_seconds": 78.73098039627075, - "variant": "all_minus_rla" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1.toml deleted file mode 100644 index a6f61b0b5..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1.toml +++ /dev/null @@ -1,32 +0,0 @@ -dataset = "homesite-insurance" -variant = "all_minus_rla" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1/report.json deleted file mode 100644 index 95758417a..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1/report.json +++ /dev/null @@ -1,181 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 19, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_minus_rla" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.9305360632758944, - "train_loss": 0.37063259875232524, - "val_score": 0.9251245714352433 - }, - { - "epoch": 1, - "test_score": 0.9385317853856436, - "train_loss": 0.26606528745456176, - "val_score": 0.935285717252423 - }, - { - "epoch": 2, - "test_score": 0.9417092028184089, - "train_loss": 0.2536956027150154, - "val_score": 0.9385187603471032 - }, - { - "epoch": 3, - "test_score": 0.9429347639616886, - "train_loss": 0.2526001357219436, - "val_score": 0.940356513335862 - }, - { - "epoch": 4, - "test_score": 0.9453012698071818, - "train_loss": 0.23765619004314595, - "val_score": 0.9422723050836483 - }, - { - "epoch": 5, - "test_score": 0.9459483004659751, - "train_loss": 0.23879074142737822, - "val_score": 0.9423621780213208 - }, - { - "epoch": 6, - "test_score": 0.9443578520626261, - "train_loss": 0.24308968741785397, - "val_score": 0.9423130125480803 - }, - { - "epoch": 7, - "test_score": 0.9453902354656645, - "train_loss": 0.23823609148914163, - "val_score": 0.942570225695371 - }, - { - "epoch": 8, - "test_score": 0.9479785030747518, - "train_loss": 0.23380982577800752, - "val_score": 0.945337735413333 - }, - { - "epoch": 9, - "test_score": 0.9481873138353728, - "train_loss": 0.23272585475986654, - "val_score": 0.9454698104739585 - }, - { - "epoch": 10, - "test_score": 0.9478268845231552, - "train_loss": 0.23764072954654694, - "val_score": 0.9451820535401805 - }, - { - "epoch": 11, - "test_score": 0.9490152840198445, - "train_loss": 0.22946590293537486, - "val_score": 0.9461689062248957 - }, - { - "epoch": 12, - "test_score": 0.9480973333594931, - "train_loss": 0.2318623957308856, - "val_score": 0.9453694661345283 - }, - { - "epoch": 13, - "test_score": 0.9469292785353847, - "train_loss": 0.22568691576069053, - "val_score": 0.9441586168901303 - }, - { - "epoch": 14, - "test_score": 0.9497359493457278, - "train_loss": 0.2304897647012364, - "val_score": 0.9472400488581012 - }, - { - "epoch": 15, - "test_score": 0.9498432058796782, - "train_loss": 0.2239592286673459, - "val_score": 0.946994010481284 - }, - { - "epoch": 16, - "test_score": 0.9489725529587292, - "train_loss": 0.22322151389988987, - "val_score": 0.9466023659881697 - }, - { - "epoch": 17, - "test_score": 0.9489942144776944, - "train_loss": 0.22959728931838816, - "val_score": 0.9468746751865251 - }, - { - "epoch": 18, - "test_score": 0.9492767323923624, - "train_loss": 0.21688801876523278, - "val_score": 0.9471568491311207 - }, - { - "epoch": 19, - "test_score": 0.9499504624136281, - "train_loss": 0.22332347401163793, - "val_score": 0.9477251798456656 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9499504624136281, - "score": 0.9499504624136281 - }, - "val": { - "roc-auc": 0.9477251798456656, - "score": 0.9477251798456656 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/1", - "seed": 1, - "task_type": "binclass", - "time_seconds": 80.56382298469543, - "variant": "all_minus_rla" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2.toml deleted file mode 100644 index 4a1e240b2..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2.toml +++ /dev/null @@ -1,32 +0,0 @@ -dataset = "homesite-insurance" -variant = "all_minus_rla" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2/report.json deleted file mode 100644 index e6ea21568..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2/report.json +++ /dev/null @@ -1,181 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 18, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_minus_rla" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.9311648634650177, - "train_loss": 0.3705499173565344, - "val_score": 0.9267300896582344 - }, - { - "epoch": 1, - "test_score": 0.9400797443510673, - "train_loss": 0.2669092017141255, - "val_score": 0.9365076005935516 - }, - { - "epoch": 2, - "test_score": 0.9425562129102363, - "train_loss": 0.25113668482411994, - "val_score": 0.9396609959732668 - }, - { - "epoch": 3, - "test_score": 0.941631702180282, - "train_loss": 0.25384827730330556, - "val_score": 0.9383607573571884 - }, - { - "epoch": 4, - "test_score": 0.9429357425356073, - "train_loss": 0.24375565146858041, - "val_score": 0.9402041460877832 - }, - { - "epoch": 5, - "test_score": 0.9448555475089664, - "train_loss": 0.2422500709241087, - "val_score": 0.9419154509656434 - }, - { - "epoch": 6, - "test_score": 0.9459835049647285, - "train_loss": 0.2398611607876691, - "val_score": 0.9434383937013894 - }, - { - "epoch": 7, - "test_score": 0.9476710134034183, - "train_loss": 0.2345308242873712, - "val_score": 0.9442095232009424 - }, - { - "epoch": 8, - "test_score": 0.9479903184487326, - "train_loss": 0.22430998127568852, - "val_score": 0.9445603019716639 - }, - { - "epoch": 9, - "test_score": 0.9462158135805571, - "train_loss": 0.24261990758505733, - "val_score": 0.9435801928345121 - }, - { - "epoch": 10, - "test_score": 0.9463019522477192, - "train_loss": 0.23785935233939778, - "val_score": 0.9441950689738309 - }, - { - "epoch": 11, - "test_score": 0.9477470280589279, - "train_loss": 0.22844018380750308, - "val_score": 0.9446325467308944 - }, - { - "epoch": 12, - "test_score": 0.9483520042015373, - "train_loss": 0.22788471349261025, - "val_score": 0.9456674307067956 - }, - { - "epoch": 13, - "test_score": 0.9496501247887126, - "train_loss": 0.2299680548635396, - "val_score": 0.9467883454687589 - }, - { - "epoch": 14, - "test_score": 0.9485763271709432, - "train_loss": 0.2274209422144023, - "val_score": 0.9462431819612933 - }, - { - "epoch": 15, - "test_score": 0.948765687264786, - "train_loss": 0.2249194474382834, - "val_score": 0.9465975391203569 - }, - { - "epoch": 16, - "test_score": 0.9479952113183261, - "train_loss": 0.22566421208056536, - "val_score": 0.9461114058323722 - }, - { - "epoch": 17, - "test_score": 0.9493630643580763, - "train_loss": 0.2237590559504249, - "val_score": 0.9475332481073184 - }, - { - "epoch": 18, - "test_score": 0.9505799995534802, - "train_loss": 0.2211470286954533, - "val_score": 0.9482112603809969 - }, - { - "epoch": 19, - "test_score": 0.9499611663209362, - "train_loss": 0.22110925899310546, - "val_score": 0.947609044878564 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9505799995534802, - "score": 0.9505799995534802 - }, - "val": { - "roc-auc": 0.9482112603809969, - "score": 0.9482112603809969 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/all_minus_rla-evaluation/2", - "seed": 2, - "task_type": "binclass", - "time_seconds": 80.3422474861145, - "variant": "all_minus_rla" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0.toml deleted file mode 100644 index 49c31352e..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0.toml +++ /dev/null @@ -1,27 +0,0 @@ -dataset = "homesite-insurance" -variant = "baseline_plr" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0/report.json deleted file mode 100644 index cd049b235..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0/report.json +++ /dev/null @@ -1,176 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 18, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "baseline_plr" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.9092551612404702, - "train_loss": 0.40265455543994905, - "val_score": 0.9069948735146987 - }, - { - "epoch": 1, - "test_score": 0.9408753853527443, - "train_loss": 0.2279508112506433, - "val_score": 0.9375135849073393 - }, - { - "epoch": 2, - "test_score": 0.9445531560869376, - "train_loss": 0.20344534069299697, - "val_score": 0.9413967956666578 - }, - { - "epoch": 3, - "test_score": 0.9475919542957172, - "train_loss": 0.19599832499569111, - "val_score": 0.9441133463211538 - }, - { - "epoch": 4, - "test_score": 0.948740111450146, - "train_loss": 0.1918256477876143, - "val_score": 0.9462975611551404 - }, - { - "epoch": 5, - "test_score": 0.9494191813439062, - "train_loss": 0.18962883813814682, - "val_score": 0.9473306163723804 - }, - { - "epoch": 6, - "test_score": 0.9488655259668092, - "train_loss": 0.18693935193798758, - "val_score": 0.948109166411886 - }, - { - "epoch": 7, - "test_score": 0.9504254211178088, - "train_loss": 0.18496789133006877, - "val_score": 0.9483060041470267 - }, - { - "epoch": 8, - "test_score": 0.9500466526054857, - "train_loss": 0.18423595116897062, - "val_score": 0.94922840188104 - }, - { - "epoch": 9, - "test_score": 0.9496331628407889, - "train_loss": 0.18239941827275535, - "val_score": 0.9492990552685507 - }, - { - "epoch": 10, - "test_score": 0.9507528205399816, - "train_loss": 0.18131620423360303, - "val_score": 0.9500436413911881 - }, - { - "epoch": 11, - "test_score": 0.9509186465351379, - "train_loss": 0.18082551035014066, - "val_score": 0.9500871007857209 - }, - { - "epoch": 12, - "test_score": 0.95122214942384, - "train_loss": 0.17972185178236527, - "val_score": 0.9503499496748476 - }, - { - "epoch": 13, - "test_score": 0.950892768691511, - "train_loss": 0.17814393598924982, - "val_score": 0.9501892738838125 - }, - { - "epoch": 14, - "test_score": 0.950397549882866, - "train_loss": 0.17763228077780116, - "val_score": 0.9497813024443802 - }, - { - "epoch": 15, - "test_score": 0.9527338407484751, - "train_loss": 0.17669161016290838, - "val_score": 0.9512763390263235 - }, - { - "epoch": 16, - "test_score": 0.952465481952729, - "train_loss": 0.1758088152516972, - "val_score": 0.9514608590168274 - }, - { - "epoch": 17, - "test_score": 0.9532766593255099, - "train_loss": 0.17514954602176494, - "val_score": 0.951688504301724 - }, - { - "epoch": 18, - "test_score": 0.9527329104991943, - "train_loss": 0.17444596791809255, - "val_score": 0.951942736924081 - }, - { - "epoch": 19, - "test_score": 0.9520153379501403, - "train_loss": 0.1736760964447802, - "val_score": 0.9514972319715476 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9527329104991943, - "score": 0.9527329104991943 - }, - "val": { - "roc-auc": 0.951942736924081, - "score": 0.951942736924081 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/0", - "seed": 0, - "task_type": "binclass", - "time_seconds": 49.509262800216675, - "variant": "baseline_plr" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1.toml deleted file mode 100644 index 2fd6f359c..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1.toml +++ /dev/null @@ -1,27 +0,0 @@ -dataset = "homesite-insurance" -variant = "baseline_plr" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1/report.json deleted file mode 100644 index d63534675..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1/report.json +++ /dev/null @@ -1,176 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 18, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "baseline_plr" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.9152291617154047, - "train_loss": 0.40116534287279304, - "val_score": 0.9128087083095999 - }, - { - "epoch": 1, - "test_score": 0.9410798227336301, - "train_loss": 0.22753327705643395, - "val_score": 0.93512569207745 - }, - { - "epoch": 2, - "test_score": 0.9450291054461962, - "train_loss": 0.20542177815328944, - "val_score": 0.940434516926454 - }, - { - "epoch": 3, - "test_score": 0.9471194118234347, - "train_loss": 0.1974578472700986, - "val_score": 0.9431667175348782 - }, - { - "epoch": 4, - "test_score": 0.9472607855517837, - "train_loss": 0.19329350929368627, - "val_score": 0.9452731485809968 - }, - { - "epoch": 5, - "test_score": 0.9492152876151975, - "train_loss": 0.18974924954501066, - "val_score": 0.9464897126962115 - }, - { - "epoch": 6, - "test_score": 0.9491863894816981, - "train_loss": 0.18798113004727798, - "val_score": 0.9469293533120763 - }, - { - "epoch": 7, - "test_score": 0.9498860094277503, - "train_loss": 0.18644553978334774, - "val_score": 0.9478204423461236 - }, - { - "epoch": 8, - "test_score": 0.9500854331274479, - "train_loss": 0.18484502827579324, - "val_score": 0.948568923373025 - }, - { - "epoch": 9, - "test_score": 0.9513454618187518, - "train_loss": 0.18316087357022545, - "val_score": 0.9494386124138916 - }, - { - "epoch": 10, - "test_score": 0.9506907596236825, - "train_loss": 0.18158665922555056, - "val_score": 0.9491904111782734 - }, - { - "epoch": 11, - "test_score": 0.951191439116417, - "train_loss": 0.18065576593984256, - "val_score": 0.9498100526406422 - }, - { - "epoch": 12, - "test_score": 0.9526199758202842, - "train_loss": 0.1797984992915934, - "val_score": 0.9495069886451847 - }, - { - "epoch": 13, - "test_score": 0.9513438308622209, - "train_loss": 0.17902052050287073, - "val_score": 0.9501305777644716 - }, - { - "epoch": 14, - "test_score": 0.9505508718779504, - "train_loss": 0.17793472490527412, - "val_score": 0.9501276587843008 - }, - { - "epoch": 15, - "test_score": 0.9529368767148602, - "train_loss": 0.17711421264843508, - "val_score": 0.950880439152478 - }, - { - "epoch": 16, - "test_score": 0.9520506995039669, - "train_loss": 0.17653414783152668, - "val_score": 0.9504092700339237 - }, - { - "epoch": 17, - "test_score": 0.9520371806864979, - "train_loss": 0.17552702819759194, - "val_score": 0.9506431840925006 - }, - { - "epoch": 18, - "test_score": 0.9530616509300706, - "train_loss": 0.1746248097582297, - "val_score": 0.9512467711639294 - }, - { - "epoch": 19, - "test_score": 0.9537073043366819, - "train_loss": 0.17431941289793362, - "val_score": 0.9502714801024816 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9530616509300706, - "score": 0.9530616509300706 - }, - "val": { - "roc-auc": 0.9512467711639294, - "score": 0.9512467711639294 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/1", - "seed": 1, - "task_type": "binclass", - "time_seconds": 49.59510111808777, - "variant": "baseline_plr" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2.toml deleted file mode 100644 index cfcdc9bd4..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2.toml +++ /dev/null @@ -1,27 +0,0 @@ -dataset = "homesite-insurance" -variant = "baseline_plr" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2/report.json deleted file mode 100644 index f86492c8b..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2/report.json +++ /dev/null @@ -1,176 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 19, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "baseline_plr" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.918776262628557, - "train_loss": 0.4011460320516066, - "val_score": 0.914550108576391 - }, - { - "epoch": 1, - "test_score": 0.9399168057530289, - "train_loss": 0.22690394493666563, - "val_score": 0.9361857126928352 - }, - { - "epoch": 2, - "test_score": 0.9448195456536869, - "train_loss": 0.2040758810260079, - "val_score": 0.9417620638329265 - }, - { - "epoch": 3, - "test_score": 0.9486874255136111, - "train_loss": 0.195921021158045, - "val_score": 0.9447484706873813 - }, - { - "epoch": 4, - "test_score": 0.9473506452160686, - "train_loss": 0.19198633405295285, - "val_score": 0.9453271760904858 - }, - { - "epoch": 5, - "test_score": 0.9478740252074842, - "train_loss": 0.18984181637113745, - "val_score": 0.9463248342770988 - }, - { - "epoch": 6, - "test_score": 0.950552043750421, - "train_loss": 0.18705180775035513, - "val_score": 0.9478500541690625 - }, - { - "epoch": 7, - "test_score": 0.949526160011188, - "train_loss": 0.1852110438726165, - "val_score": 0.9479885210928937 - }, - { - "epoch": 8, - "test_score": 0.9500179477705382, - "train_loss": 0.1838859125971794, - "val_score": 0.9489083691153118 - }, - { - "epoch": 9, - "test_score": 0.951933753880106, - "train_loss": 0.182511460239237, - "val_score": 0.949106499290468 - }, - { - "epoch": 10, - "test_score": 0.9505244141386678, - "train_loss": 0.18200897046110848, - "val_score": 0.9494714861092504 - }, - { - "epoch": 11, - "test_score": 0.951459894561439, - "train_loss": 0.18012812530452554, - "val_score": 0.9500796011167878 - }, - { - "epoch": 12, - "test_score": 0.9524452822540621, - "train_loss": 0.1792988423596729, - "val_score": 0.9499561423229318 - }, - { - "epoch": 13, - "test_score": 0.9514597375063656, - "train_loss": 0.1785095995122736, - "val_score": 0.9502315639278549 - }, - { - "epoch": 14, - "test_score": 0.9517478489978773, - "train_loss": 0.17744325155561622, - "val_score": 0.950572829636694 - }, - { - "epoch": 15, - "test_score": 0.9520442844082779, - "train_loss": 0.1771064200184562, - "val_score": 0.9507174422446804 - }, - { - "epoch": 16, - "test_score": 0.9534308632452372, - "train_loss": 0.17622807310386138, - "val_score": 0.9508445145953139 - }, - { - "epoch": 17, - "test_score": 0.9525583498257268, - "train_loss": 0.17589684643528677, - "val_score": 0.950965230251178 - }, - { - "epoch": 18, - "test_score": 0.9526653768176468, - "train_loss": 0.17428164116360925, - "val_score": 0.9513501136125109 - }, - { - "epoch": 19, - "test_score": 0.9538975705174861, - "train_loss": 0.17425130158662797, - "val_score": 0.9515441114964615 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9538975705174861, - "score": 0.9538975705174861 - }, - "val": { - "roc-auc": 0.9515441114964615, - "score": 0.9515441114964615 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/baseline_plr-evaluation/2", - "seed": 2, - "task_type": "binclass", - "time_seconds": 52.342146158218384, - "variant": "baseline_plr" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0.toml deleted file mode 100644 index 6f754cff6..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0.toml +++ /dev/null @@ -1,28 +0,0 @@ -dataset = "homesite-insurance" -variant = "cf_fisd_only" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0/report.json deleted file mode 100644 index d0b26f2fa..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0/report.json +++ /dev/null @@ -1,177 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 17, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "cf_fisd_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.9356782275953207, - "train_loss": 0.34595361920920287, - "val_score": 0.932453427275728 - }, - { - "epoch": 1, - "test_score": 0.9450704230116513, - "train_loss": 0.2139470558274876, - "val_score": 0.9414894732870849 - }, - { - "epoch": 2, - "test_score": 0.9472626218880262, - "train_loss": 0.20079242763194172, - "val_score": 0.943445313091132 - }, - { - "epoch": 3, - "test_score": 0.9502677378241484, - "train_loss": 0.19390834854407743, - "val_score": 0.9455957662267558 - }, - { - "epoch": 4, - "test_score": 0.9512641556153858, - "train_loss": 0.1904226701367985, - "val_score": 0.9471411200482117 - }, - { - "epoch": 5, - "test_score": 0.9516697322206167, - "train_loss": 0.1884030981497331, - "val_score": 0.9477306837058675 - }, - { - "epoch": 6, - "test_score": 0.951924825903243, - "train_loss": 0.18619168563322588, - "val_score": 0.9484983930750396 - }, - { - "epoch": 7, - "test_score": 0.9525656226837398, - "train_loss": 0.18428844687613574, - "val_score": 0.9486409219532052 - }, - { - "epoch": 8, - "test_score": 0.952576954811341, - "train_loss": 0.18353629532185467, - "val_score": 0.9492243926793594 - }, - { - "epoch": 9, - "test_score": 0.9527248523658148, - "train_loss": 0.18191406997767362, - "val_score": 0.9493485196734955 - }, - { - "epoch": 10, - "test_score": 0.9529742920658001, - "train_loss": 0.180889469249682, - "val_score": 0.9500480550298802 - }, - { - "epoch": 11, - "test_score": 0.9533969514305494, - "train_loss": 0.18059069622646678, - "val_score": 0.9499987664671146 - }, - { - "epoch": 12, - "test_score": 0.9535607719532305, - "train_loss": 0.17952959659424694, - "val_score": 0.950490429991628 - }, - { - "epoch": 13, - "test_score": 0.9526063482723798, - "train_loss": 0.17814569906754926, - "val_score": 0.9500473164927284 - }, - { - "epoch": 14, - "test_score": 0.9527289841223603, - "train_loss": 0.1777255663817579, - "val_score": 0.9496370063523164 - }, - { - "epoch": 15, - "test_score": 0.9539749141005397, - "train_loss": 0.1767724638635462, - "val_score": 0.9511459608427252 - }, - { - "epoch": 16, - "test_score": 0.9541068403621681, - "train_loss": 0.17599785151806743, - "val_score": 0.9512804889017473 - }, - { - "epoch": 17, - "test_score": 0.9543305592735997, - "train_loss": 0.17548072730953043, - "val_score": 0.9515030083871268 - }, - { - "epoch": 18, - "test_score": 0.9539477556540068, - "train_loss": 0.17482344752008264, - "val_score": 0.9514199141654539 - }, - { - "epoch": 19, - "test_score": 0.9534808913266842, - "train_loss": 0.17422030784867026, - "val_score": 0.9511812171996087 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9543305592735997, - "score": 0.9543305592735997 - }, - "val": { - "roc-auc": 0.9515030083871268, - "score": 0.9515030083871268 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/0", - "seed": 0, - "task_type": "binclass", - "time_seconds": 48.676718950271606, - "variant": "cf_fisd_only" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1.toml deleted file mode 100644 index c0b676360..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1.toml +++ /dev/null @@ -1,28 +0,0 @@ -dataset = "homesite-insurance" -variant = "cf_fisd_only" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1/report.json deleted file mode 100644 index 62a65e68f..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1/report.json +++ /dev/null @@ -1,177 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 18, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "cf_fisd_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.9341219205486897, - "train_loss": 0.3445870617573911, - "val_score": 0.9301370054994816 - }, - { - "epoch": 1, - "test_score": 0.9455012854836944, - "train_loss": 0.21479442458261144, - "val_score": 0.9419169807926006 - }, - { - "epoch": 2, - "test_score": 0.9477954010215248, - "train_loss": 0.20204473598436876, - "val_score": 0.9440465087089268 - }, - { - "epoch": 3, - "test_score": 0.9502180359340077, - "train_loss": 0.1942431166768074, - "val_score": 0.9464708799988436 - }, - { - "epoch": 4, - "test_score": 0.950256188235676, - "train_loss": 0.19089977063915947, - "val_score": 0.9466997913474287 - }, - { - "epoch": 5, - "test_score": 0.9516093505854868, - "train_loss": 0.18810367069461129, - "val_score": 0.9472306852620709 - }, - { - "epoch": 6, - "test_score": 0.9514332314424452, - "train_loss": 0.18654819618571888, - "val_score": 0.9474986775349324 - }, - { - "epoch": 7, - "test_score": 0.9522986048966968, - "train_loss": 0.1853742083365267, - "val_score": 0.9484116852965887 - }, - { - "epoch": 8, - "test_score": 0.9523285178475936, - "train_loss": 0.18406797484918075, - "val_score": 0.9490725529578176 - }, - { - "epoch": 9, - "test_score": 0.9534142516509388, - "train_loss": 0.1827931050549854, - "val_score": 0.9496698185029124 - }, - { - "epoch": 10, - "test_score": 0.9527738656298648, - "train_loss": 0.1812525972723961, - "val_score": 0.9496054163048641 - }, - { - "epoch": 11, - "test_score": 0.9529002466392873, - "train_loss": 0.1802022703669288, - "val_score": 0.9498561408757505 - }, - { - "epoch": 12, - "test_score": 0.9535008010775234, - "train_loss": 0.17959909669377588, - "val_score": 0.9494651645829163 - }, - { - "epoch": 13, - "test_score": 0.9532634787805064, - "train_loss": 0.17891152690757406, - "val_score": 0.9501905047790651 - }, - { - "epoch": 14, - "test_score": 0.9519700698455323, - "train_loss": 0.17768442129546946, - "val_score": 0.9499346456165526 - }, - { - "epoch": 15, - "test_score": 0.9539713018338523, - "train_loss": 0.17704763520847666, - "val_score": 0.950807570153512 - }, - { - "epoch": 16, - "test_score": 0.9529998074746422, - "train_loss": 0.1763513352383267, - "val_score": 0.950307307946447 - }, - { - "epoch": 17, - "test_score": 0.9530448339560608, - "train_loss": 0.17554833618077365, - "val_score": 0.950576091509114 - }, - { - "epoch": 18, - "test_score": 0.9539302138104276, - "train_loss": 0.1747353514487093, - "val_score": 0.9512915581669135 - }, - { - "epoch": 19, - "test_score": 0.95444112604525, - "train_loss": 0.17438724718310616, - "val_score": 0.9504254123459533 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9539302138104276, - "score": 0.9539302138104276 - }, - "val": { - "roc-auc": 0.9512915581669135, - "score": 0.9512915581669135 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/1", - "seed": 1, - "task_type": "binclass", - "time_seconds": 51.1213116645813, - "variant": "cf_fisd_only" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2.toml deleted file mode 100644 index f2eb7bc69..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2.toml +++ /dev/null @@ -1,28 +0,0 @@ -dataset = "homesite-insurance" -variant = "cf_fisd_only" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2/report.json deleted file mode 100644 index 40dcc2465..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2/report.json +++ /dev/null @@ -1,177 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 19, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "cf_fisd_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.9368268196714911, - "train_loss": 0.3421837103637782, - "val_score": 0.9326621695263851 - }, - { - "epoch": 1, - "test_score": 0.9439968428547529, - "train_loss": 0.214590031450445, - "val_score": 0.9407243400058346 - }, - { - "epoch": 2, - "test_score": 0.9473855356046752, - "train_loss": 0.20220571661537343, - "val_score": 0.9441511875580686 - }, - { - "epoch": 3, - "test_score": 0.9500893957477605, - "train_loss": 0.19425439482385462, - "val_score": 0.9462104753160047 - }, - { - "epoch": 4, - "test_score": 0.9496412572176471, - "train_loss": 0.1903872162103653, - "val_score": 0.946274886306162 - }, - { - "epoch": 5, - "test_score": 0.9502430439341512, - "train_loss": 0.188402750546282, - "val_score": 0.9469396224953281 - }, - { - "epoch": 6, - "test_score": 0.952653790985696, - "train_loss": 0.18608629364858975, - "val_score": 0.9482615512441828 - }, - { - "epoch": 7, - "test_score": 0.9516714115017857, - "train_loss": 0.18428249873898245, - "val_score": 0.9480207002116454 - }, - { - "epoch": 8, - "test_score": 0.9526286863362763, - "train_loss": 0.18304257894104178, - "val_score": 0.949256809185053 - }, - { - "epoch": 9, - "test_score": 0.9536667358031155, - "train_loss": 0.18201136128468948, - "val_score": 0.9492468037650695 - }, - { - "epoch": 10, - "test_score": 0.9526991678207396, - "train_loss": 0.18154333640228618, - "val_score": 0.9496464139088913 - }, - { - "epoch": 11, - "test_score": 0.9533547640216111, - "train_loss": 0.1797881559892134, - "val_score": 0.9503523675048084 - }, - { - "epoch": 12, - "test_score": 0.9540047062398416, - "train_loss": 0.17908011282032185, - "val_score": 0.9503211643101501 - }, - { - "epoch": 13, - "test_score": 0.9527204306614416, - "train_loss": 0.17852983230894262, - "val_score": 0.9501864076562951 - }, - { - "epoch": 14, - "test_score": 0.9532465409949018, - "train_loss": 0.17740027660673316, - "val_score": 0.9507598993387929 - }, - { - "epoch": 15, - "test_score": 0.9535459966951747, - "train_loss": 0.17713124508207495, - "val_score": 0.9510151518457881 - }, - { - "epoch": 16, - "test_score": 0.9538396534388924, - "train_loss": 0.17623664411631498, - "val_score": 0.9507649284251114 - }, - { - "epoch": 17, - "test_score": 0.9536572279305969, - "train_loss": 0.17600064846602353, - "val_score": 0.9508994740683517 - }, - { - "epoch": 18, - "test_score": 0.9538222082445892, - "train_loss": 0.17448357275941156, - "val_score": 0.9514048005301712 - }, - { - "epoch": 19, - "test_score": 0.9548333529692444, - "train_loss": 0.17447115494446322, - "val_score": 0.9519185850007992 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9548333529692444, - "score": 0.9548333529692444 - }, - "val": { - "roc-auc": 0.9519185850007992, - "score": 0.9519185850007992 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/cf_fisd_only-evaluation/2", - "seed": 2, - "task_type": "binclass", - "time_seconds": 51.285852909088135, - "variant": "cf_fisd_only" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0.toml deleted file mode 100644 index a9656bec3..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "homesite-insurance" -variant = "esam_only" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0/report.json deleted file mode 100644 index b764b9767..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0/report.json +++ /dev/null @@ -1,178 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 18, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "esam_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.908729026744691, - "train_loss": 0.4032717401331121, - "val_score": 0.906570592713167 - }, - { - "epoch": 1, - "test_score": 0.9408769679846375, - "train_loss": 0.22863993658260864, - "val_score": 0.9373667566878979 - }, - { - "epoch": 2, - "test_score": 0.9445106787301715, - "train_loss": 0.20407469245520504, - "val_score": 0.941151777174479 - }, - { - "epoch": 3, - "test_score": 0.9475020342256348, - "train_loss": 0.19658909020098772, - "val_score": 0.9440126678815838 - }, - { - "epoch": 4, - "test_score": 0.9488865955089594, - "train_loss": 0.19217103773897345, - "val_score": 0.9462030196076161 - }, - { - "epoch": 5, - "test_score": 0.9495634062258951, - "train_loss": 0.19011524509299885, - "val_score": 0.9472272739237988 - }, - { - "epoch": 6, - "test_score": 0.9486842602498249, - "train_loss": 0.18731397566470234, - "val_score": 0.9479469871702204 - }, - { - "epoch": 7, - "test_score": 0.9506030141623083, - "train_loss": 0.18545044728300789, - "val_score": 0.9481834773167194 - }, - { - "epoch": 8, - "test_score": 0.949899769868409, - "train_loss": 0.18469505269419062, - "val_score": 0.9491123987955724 - }, - { - "epoch": 9, - "test_score": 0.9496426103075098, - "train_loss": 0.1828580309044231, - "val_score": 0.9492144839725745 - }, - { - "epoch": 10, - "test_score": 0.9505282559473858, - "train_loss": 0.18168031546202573, - "val_score": 0.9500062925123746 - }, - { - "epoch": 11, - "test_score": 0.9508295963085389, - "train_loss": 0.18124051933938806, - "val_score": 0.9500708969289285 - }, - { - "epoch": 12, - "test_score": 0.9511067743507127, - "train_loss": 0.179936251857064, - "val_score": 0.9503749104721527 - }, - { - "epoch": 13, - "test_score": 0.9508392612361309, - "train_loss": 0.17850419987331737, - "val_score": 0.9502711284181236 - }, - { - "epoch": 14, - "test_score": 0.9506935503715248, - "train_loss": 0.1779331474141641, - "val_score": 0.9501258651940752 - }, - { - "epoch": 15, - "test_score": 0.9527863213052987, - "train_loss": 0.17700313343243165, - "val_score": 0.9514039740719301 - }, - { - "epoch": 16, - "test_score": 0.9525207290950757, - "train_loss": 0.17610342638059095, - "val_score": 0.9515501956358539 - }, - { - "epoch": 17, - "test_score": 0.9533450266070624, - "train_loss": 0.1753809163516218, - "val_score": 0.9517171665768963 - }, - { - "epoch": 18, - "test_score": 0.9528578176071593, - "train_loss": 0.1747145946730267, - "val_score": 0.9520053367397949 - }, - { - "epoch": 19, - "test_score": 0.9522033207918011, - "train_loss": 0.17411214180968024, - "val_score": 0.9517029585288354 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9528578176071593, - "score": 0.9528578176071593 - }, - "val": { - "roc-auc": 0.9520053367397949, - "score": 0.9520053367397949 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/0", - "seed": 0, - "task_type": "binclass", - "time_seconds": 77.46188020706177, - "variant": "esam_only" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1.toml deleted file mode 100644 index 9cf6fb745..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "homesite-insurance" -variant = "esam_only" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1/report.json deleted file mode 100644 index 46458166c..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1/report.json +++ /dev/null @@ -1,178 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 18, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "esam_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.9147756953939565, - "train_loss": 0.40172804539853874, - "val_score": 0.9124704495020212 - }, - { - "epoch": 1, - "test_score": 0.9407769722275406, - "train_loss": 0.2284326516769149, - "val_score": 0.9349377167881294 - }, - { - "epoch": 2, - "test_score": 0.9448971187787707, - "train_loss": 0.2062601380727508, - "val_score": 0.9402407915978808 - }, - { - "epoch": 3, - "test_score": 0.9468855568191913, - "train_loss": 0.1979634551839395, - "val_score": 0.9429383864654839 - }, - { - "epoch": 4, - "test_score": 0.9470005090517362, - "train_loss": 0.19388176703994925, - "val_score": 0.9450562296690174 - }, - { - "epoch": 5, - "test_score": 0.9489158319149248, - "train_loss": 0.1902060573751276, - "val_score": 0.9463056059348284 - }, - { - "epoch": 6, - "test_score": 0.9488005051664354, - "train_loss": 0.18836823295463215, - "val_score": 0.9467129091739801 - }, - { - "epoch": 7, - "test_score": 0.9496745287308819, - "train_loss": 0.1869170370427045, - "val_score": 0.9476899938256536 - }, - { - "epoch": 8, - "test_score": 0.9498384579839986, - "train_loss": 0.1851530686020851, - "val_score": 0.948475076402108 - }, - { - "epoch": 9, - "test_score": 0.9512796194995327, - "train_loss": 0.18356213136152788, - "val_score": 0.9493400177041422 - }, - { - "epoch": 10, - "test_score": 0.9506040048173865, - "train_loss": 0.18194633613933217, - "val_score": 0.9490656775286198 - }, - { - "epoch": 11, - "test_score": 0.9510479511851571, - "train_loss": 0.18100393509322946, - "val_score": 0.949754521680523 - }, - { - "epoch": 12, - "test_score": 0.9524267135119263, - "train_loss": 0.1802269911224192, - "val_score": 0.94947667345353 - }, - { - "epoch": 13, - "test_score": 0.9511907988149638, - "train_loss": 0.1793379318985072, - "val_score": 0.9500892372681953 - }, - { - "epoch": 14, - "test_score": 0.9504332013845201, - "train_loss": 0.1783846991983327, - "val_score": 0.9500476242165418 - }, - { - "epoch": 15, - "test_score": 0.9528141804590822, - "train_loss": 0.17759880057790062, - "val_score": 0.9508749528764942 - }, - { - "epoch": 16, - "test_score": 0.9519126601756369, - "train_loss": 0.17679053274067966, - "val_score": 0.9503236700612003 - }, - { - "epoch": 17, - "test_score": 0.9519582786338702, - "train_loss": 0.1758375184102492, - "val_score": 0.9505821668563976 - }, - { - "epoch": 18, - "test_score": 0.9530825030113498, - "train_loss": 0.17506359707225452, - "val_score": 0.9512828188106186 - }, - { - "epoch": 19, - "test_score": 0.9535977644635883, - "train_loss": 0.17463966445489362, - "val_score": 0.9501736415141018 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9530825030113498, - "score": 0.9530825030113498 - }, - "val": { - "roc-auc": 0.9512828188106186, - "score": 0.9512828188106186 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/1", - "seed": 1, - "task_type": "binclass", - "time_seconds": 79.07510232925415, - "variant": "esam_only" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2.toml deleted file mode 100644 index 91757e813..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "homesite-insurance" -variant = "esam_only" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2/report.json deleted file mode 100644 index 0da573087..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2/report.json +++ /dev/null @@ -1,178 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 19, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "esam_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.918368837605924, - "train_loss": 0.4018080147829923, - "val_score": 0.914369043884704 - }, - { - "epoch": 1, - "test_score": 0.9396808727893531, - "train_loss": 0.2275132194161415, - "val_score": 0.9359316471205482 - }, - { - "epoch": 2, - "test_score": 0.9447300000995488, - "train_loss": 0.20466342500664972, - "val_score": 0.941619904223337 - }, - { - "epoch": 3, - "test_score": 0.9486153493160954, - "train_loss": 0.19632872586900538, - "val_score": 0.9446383407306913 - }, - { - "epoch": 4, - "test_score": 0.9472420114299367, - "train_loss": 0.19241265126250007, - "val_score": 0.9452525047091854 - }, - { - "epoch": 5, - "test_score": 0.9477903873403365, - "train_loss": 0.19022366553544998, - "val_score": 0.9463075050303612 - }, - { - "epoch": 6, - "test_score": 0.9503154221606545, - "train_loss": 0.18738075386394154, - "val_score": 0.9476912071366883 - }, - { - "epoch": 7, - "test_score": 0.9493462353029068, - "train_loss": 0.18546505800702356, - "val_score": 0.947911519802722 - }, - { - "epoch": 8, - "test_score": 0.9497630473864636, - "train_loss": 0.18423645306717265, - "val_score": 0.9488656218816038 - }, - { - "epoch": 9, - "test_score": 0.9517127894730382, - "train_loss": 0.18286606940356168, - "val_score": 0.9490535971709243 - }, - { - "epoch": 10, - "test_score": 0.9501892102874651, - "train_loss": 0.18241622298955917, - "val_score": 0.9494314995977523 - }, - { - "epoch": 11, - "test_score": 0.951262440090738, - "train_loss": 0.18050547662106428, - "val_score": 0.9499718274452961 - }, - { - "epoch": 12, - "test_score": 0.9523709831231999, - "train_loss": 0.17971274432810871, - "val_score": 0.9499982301484686 - }, - { - "epoch": 13, - "test_score": 0.9512103582121777, - "train_loss": 0.1789512035521594, - "val_score": 0.950264877228661 - }, - { - "epoch": 14, - "test_score": 0.9516913091714654, - "train_loss": 0.17783631357279692, - "val_score": 0.9505823339064675 - }, - { - "epoch": 15, - "test_score": 0.9519843739383681, - "train_loss": 0.17752045812931927, - "val_score": 0.9507157629518711 - }, - { - "epoch": 16, - "test_score": 0.9534914985847162, - "train_loss": 0.17668290869756179, - "val_score": 0.9509019094825304 - }, - { - "epoch": 17, - "test_score": 0.9527285492006186, - "train_loss": 0.1762797315012325, - "val_score": 0.9509802647574802 - }, - { - "epoch": 18, - "test_score": 0.9528286295258319, - "train_loss": 0.17472596371715718, - "val_score": 0.9513271750002642 - }, - { - "epoch": 19, - "test_score": 0.9538855376826346, - "train_loss": 0.1745066287842664, - "val_score": 0.9515687469857353 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9538855376826346, - "score": 0.9538855376826346 - }, - "val": { - "roc-auc": 0.9515687469857353, - "score": 0.9515687469857353 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/esam_only-evaluation/2", - "seed": 2, - "task_type": "binclass", - "time_seconds": 81.17848920822144, - "variant": "esam_only" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0.toml deleted file mode 100644 index 8d18b8743..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "homesite-insurance" -variant = "mfb_only" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0/report.json deleted file mode 100644 index ec843e55b..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0/report.json +++ /dev/null @@ -1,178 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 19, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "mfb_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.8930970574258102, - "train_loss": 0.41482925062829795, - "val_score": 0.8924687005757092 - }, - { - "epoch": 1, - "test_score": 0.9331467414358352, - "train_loss": 0.2881732252511111, - "val_score": 0.929492192229193 - }, - { - "epoch": 2, - "test_score": 0.9378768898920224, - "train_loss": 0.26185889582742344, - "val_score": 0.9349895374782722 - }, - { - "epoch": 3, - "test_score": 0.9417591946563775, - "train_loss": 0.25147163529287686, - "val_score": 0.9388787268716761 - }, - { - "epoch": 4, - "test_score": 0.9420589523856374, - "train_loss": 0.24695172946561467, - "val_score": 0.9379653234651171 - }, - { - "epoch": 5, - "test_score": 0.9444818651647885, - "train_loss": 0.2433497359806841, - "val_score": 0.9416113231250033 - }, - { - "epoch": 6, - "test_score": 0.945757369821399, - "train_loss": 0.23861975249919024, - "val_score": 0.9428739227226728 - }, - { - "epoch": 7, - "test_score": 0.9469385085412347, - "train_loss": 0.24340633235194467, - "val_score": 0.9440592660590111 - }, - { - "epoch": 8, - "test_score": 0.9472159886123956, - "train_loss": 0.23544722199440002, - "val_score": 0.9436778204122771 - }, - { - "epoch": 9, - "test_score": 0.9472175591631293, - "train_loss": 0.23263336379419675, - "val_score": 0.9444757746362322 - }, - { - "epoch": 10, - "test_score": 0.9475347500055331, - "train_loss": 0.2365114538507028, - "val_score": 0.9451331694144264 - }, - { - "epoch": 11, - "test_score": 0.9489926560081199, - "train_loss": 0.23266613673080097, - "val_score": 0.9459791637216766 - }, - { - "epoch": 12, - "test_score": 0.9478042444302713, - "train_loss": 0.23365027592940765, - "val_score": 0.9451919358706387 - }, - { - "epoch": 13, - "test_score": 0.9493018491229417, - "train_loss": 0.22537394247271797, - "val_score": 0.9461747793536734 - }, - { - "epoch": 14, - "test_score": 0.94857972197676, - "train_loss": 0.23251519352197647, - "val_score": 0.9461889874017343 - }, - { - "epoch": 15, - "test_score": 0.9505958621158904, - "train_loss": 0.22507083226333965, - "val_score": 0.9471473624455652 - }, - { - "epoch": 16, - "test_score": 0.949072403741912, - "train_loss": 0.22468203807418996, - "val_score": 0.9466362507760575 - }, - { - "epoch": 17, - "test_score": 0.9498181737172153, - "train_loss": 0.2172338836572387, - "val_score": 0.9474565105804152 - }, - { - "epoch": 18, - "test_score": 0.9495097054719629, - "train_loss": 0.2258679667657072, - "val_score": 0.9473183865488329 - }, - { - "epoch": 19, - "test_score": 0.9503995432741815, - "train_loss": 0.21923035166480326, - "val_score": 0.947966382562561 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9503995432741815, - "score": 0.9503995432741815 - }, - "val": { - "roc-auc": 0.947966382562561, - "score": 0.947966382562561 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/0", - "seed": 0, - "task_type": "binclass", - "time_seconds": 49.44327640533447, - "variant": "mfb_only" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1.toml deleted file mode 100644 index bb63e7362..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "homesite-insurance" -variant = "mfb_only" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1/report.json deleted file mode 100644 index dba6eeee0..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1/report.json +++ /dev/null @@ -1,178 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 18, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "mfb_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.90697903454912, - "train_loss": 0.4116514666513963, - "val_score": 0.9035914305564858 - }, - { - "epoch": 1, - "test_score": 0.9317266253001322, - "train_loss": 0.2784177227453752, - "val_score": 0.9259784610563293 - }, - { - "epoch": 2, - "test_score": 0.9383684722716611, - "train_loss": 0.2549135125496171, - "val_score": 0.9332698273442572 - }, - { - "epoch": 3, - "test_score": 0.9389794889940154, - "train_loss": 0.2599698143926534, - "val_score": 0.9349251968249865 - }, - { - "epoch": 4, - "test_score": 0.9427043762502187, - "train_loss": 0.24525119445540688, - "val_score": 0.9390834335443257 - }, - { - "epoch": 5, - "test_score": 0.9455191535185797, - "train_loss": 0.24000001563267273, - "val_score": 0.9416237551670565 - }, - { - "epoch": 6, - "test_score": 0.9438849229932397, - "train_loss": 0.23375196429816159, - "val_score": 0.9414440620443656 - }, - { - "epoch": 7, - "test_score": 0.9461117223103932, - "train_loss": 0.2322191279042851, - "val_score": 0.9432047873666256 - }, - { - "epoch": 8, - "test_score": 0.9456184606495858, - "train_loss": 0.23114837624809959, - "val_score": 0.9428914541879162 - }, - { - "epoch": 9, - "test_score": 0.9455694473855355, - "train_loss": 0.23292614072561263, - "val_score": 0.9433439488670636 - }, - { - "epoch": 10, - "test_score": 0.9469385568658729, - "train_loss": 0.2312833772464232, - "val_score": 0.9443709814896741 - }, - { - "epoch": 11, - "test_score": 0.9480216448952896, - "train_loss": 0.23427447879856284, - "val_score": 0.9445003046201987 - }, - { - "epoch": 12, - "test_score": 0.947081042060895, - "train_loss": 0.2386595448309725, - "val_score": 0.9442177350307004 - }, - { - "epoch": 13, - "test_score": 0.9476110062842326, - "train_loss": 0.22934468537569047, - "val_score": 0.945519327631557 - }, - { - "epoch": 14, - "test_score": 0.9486105893392562, - "train_loss": 0.2297188869931481, - "val_score": 0.9461029829919994 - }, - { - "epoch": 15, - "test_score": 0.9489645189876684, - "train_loss": 0.22718335647474636, - "val_score": 0.9460358376559592 - }, - { - "epoch": 16, - "test_score": 0.948593832771044, - "train_loss": 0.2275947011329911, - "val_score": 0.9461941044091424 - }, - { - "epoch": 17, - "test_score": 0.9492380243573573, - "train_loss": 0.21871563467112454, - "val_score": 0.9470392370897156 - }, - { - "epoch": 18, - "test_score": 0.94973945288198, - "train_loss": 0.2260531865737655, - "val_score": 0.947404892108778 - }, - { - "epoch": 19, - "test_score": 0.9498296266564115, - "train_loss": 0.2279391365972432, - "val_score": 0.9473486665720519 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.94973945288198, - "score": 0.94973945288198 - }, - "val": { - "roc-auc": 0.947404892108778, - "score": 0.947404892108778 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/1", - "seed": 1, - "task_type": "binclass", - "time_seconds": 48.79517936706543, - "variant": "mfb_only" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2.toml deleted file mode 100644 index bf20c6844..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "homesite-insurance" -variant = "mfb_only" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2/report.json deleted file mode 100644 index 6b2883abf..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2/report.json +++ /dev/null @@ -1,178 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 17, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "mfb_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.9094044602094448, - "train_loss": 0.41223958161744206, - "val_score": 0.9054979114608956 - }, - { - "epoch": 1, - "test_score": 0.933507545263997, - "train_loss": 0.28018381636251105, - "val_score": 0.9294203958675189 - }, - { - "epoch": 2, - "test_score": 0.9390422868610431, - "train_loss": 0.25977433242581105, - "val_score": 0.9353415559363555 - }, - { - "epoch": 3, - "test_score": 0.9399967951100104, - "train_loss": 0.2551980449394746, - "val_score": 0.9367911197308166 - }, - { - "epoch": 4, - "test_score": 0.9444567242718902, - "train_loss": 0.24537605155598033, - "val_score": 0.941659556634695 - }, - { - "epoch": 5, - "test_score": 0.9435413589796697, - "train_loss": 0.24892445301467722, - "val_score": 0.940676598854244 - }, - { - "epoch": 6, - "test_score": 0.9447349050503016, - "train_loss": 0.2462358137423342, - "val_score": 0.9421798824343809 - }, - { - "epoch": 7, - "test_score": 0.946477829767571, - "train_loss": 0.2382623243060979, - "val_score": 0.9432349706766461 - }, - { - "epoch": 8, - "test_score": 0.9459221809991583, - "train_loss": 0.23777699958194387, - "val_score": 0.9431933048723389 - }, - { - "epoch": 9, - "test_score": 0.9456133261568024, - "train_loss": 0.23771833492950958, - "val_score": 0.9430304574303932 - }, - { - "epoch": 10, - "test_score": 0.9482286555631468, - "train_loss": 0.23080746843056246, - "val_score": 0.9454318109790827 - }, - { - "epoch": 11, - "test_score": 0.9440564029710373, - "train_loss": 0.24243825890801168, - "val_score": 0.9417709878235093 - }, - { - "epoch": 12, - "test_score": 0.9483229127694859, - "train_loss": 0.23124837455424396, - "val_score": 0.9450030022414426 - }, - { - "epoch": 13, - "test_score": 0.9462020652210578, - "train_loss": 0.23264152123169465, - "val_score": 0.9434262957594763 - }, - { - "epoch": 14, - "test_score": 0.9489400063150637, - "train_loss": 0.2291149674491449, - "val_score": 0.9462765040542086 - }, - { - "epoch": 15, - "test_score": 0.9497078244064356, - "train_loss": 0.22526589808138933, - "val_score": 0.9460425812035226 - }, - { - "epoch": 16, - "test_score": 0.9500673476316916, - "train_loss": 0.21986289972608739, - "val_score": 0.9468494154575158 - }, - { - "epoch": 17, - "test_score": 0.9499664578687927, - "train_loss": 0.22433200790123506, - "val_score": 0.947465100470858 - }, - { - "epoch": 18, - "test_score": 0.9495361752924052, - "train_loss": 0.23154993870041587, - "val_score": 0.9463413282734864 - }, - { - "epoch": 19, - "test_score": 0.9493388053898209, - "train_loss": 0.2206502684138038, - "val_score": 0.946505186807961 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9499664578687927, - "score": 0.9499664578687927 - }, - "val": { - "roc-auc": 0.947465100470858, - "score": 0.947465100470858 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/mfb_only-evaluation/2", - "seed": 2, - "task_type": "binclass", - "time_seconds": 52.592193603515625, - "variant": "mfb_only" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0.toml deleted file mode 100644 index a58c4224a..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0.toml +++ /dev/null @@ -1,30 +0,0 @@ -dataset = "homesite-insurance" -variant = "rla_only" -seed = 0 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0/report.json deleted file mode 100644 index 260031bc0..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0/report.json +++ /dev/null @@ -1,179 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 19, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "rla_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.9070250516856164, - "train_loss": 0.4060501190749082, - "val_score": 0.9046584057300496 - }, - { - "epoch": 1, - "test_score": 0.940496737652016, - "train_loss": 0.22928613221103494, - "val_score": 0.9369488765416809 - }, - { - "epoch": 2, - "test_score": 0.9447281154386683, - "train_loss": 0.2031014254147356, - "val_score": 0.9417829099232438 - }, - { - "epoch": 3, - "test_score": 0.944822408888486, - "train_loss": 0.1961885085160082, - "val_score": 0.9434031109681789 - }, - { - "epoch": 4, - "test_score": 0.9469906629067522, - "train_loss": 0.19204557131637226, - "val_score": 0.9441429757283105 - }, - { - "epoch": 5, - "test_score": 0.9480114121532017, - "train_loss": 0.18933298194950277, - "val_score": 0.9470159907536556 - }, - { - "epoch": 6, - "test_score": 0.9461172675625991, - "train_loss": 0.18728381584991108, - "val_score": 0.9473758781492477 - }, - { - "epoch": 7, - "test_score": 0.9469529334456656, - "train_loss": 0.18507160883058202, - "val_score": 0.9480603526230034 - }, - { - "epoch": 8, - "test_score": 0.9505410257329664, - "train_loss": 0.183949869193814, - "val_score": 0.949131125987633 - }, - { - "epoch": 9, - "test_score": 0.9476778271773705, - "train_loss": 0.18238650302995335, - "val_score": 0.948899963859157 - }, - { - "epoch": 10, - "test_score": 0.9484377200281172, - "train_loss": 0.1812233101237904, - "val_score": 0.9493756433296019 - }, - { - "epoch": 11, - "test_score": 0.9511458689828213, - "train_loss": 0.18025330684401772, - "val_score": 0.9503379660303507 - }, - { - "epoch": 12, - "test_score": 0.9503413000042816, - "train_loss": 0.1791025774045424, - "val_score": 0.9504818313090763 - }, - { - "epoch": 13, - "test_score": 0.9496615535655901, - "train_loss": 0.17824541519988668, - "val_score": 0.9503709891915616 - }, - { - "epoch": 14, - "test_score": 0.9510270145357611, - "train_loss": 0.17772333513606678, - "val_score": 0.950913761245393 - }, - { - "epoch": 15, - "test_score": 0.950622839345031, - "train_loss": 0.17640978937799282, - "val_score": 0.9509986754336184 - }, - { - "epoch": 16, - "test_score": 0.9512123153600153, - "train_loss": 0.1757170327685096, - "val_score": 0.9515215245685725 - }, - { - "epoch": 17, - "test_score": 0.9499410028657478, - "train_loss": 0.17498180933974006, - "val_score": 0.9509867885023199 - }, - { - "epoch": 18, - "test_score": 0.9513152468388678, - "train_loss": 0.17456490248441697, - "val_score": 0.9513378134520921 - }, - { - "epoch": 19, - "test_score": 0.9511202086000653, - "train_loss": 0.17379635247317227, - "val_score": 0.9516870623958563 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9511202086000653, - "score": 0.9511202086000653 - }, - "val": { - "roc-auc": 0.9516870623958563, - "score": 0.9516870623958563 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/0", - "seed": 0, - "task_type": "binclass", - "time_seconds": 58.163018226623535, - "variant": "rla_only" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1.toml deleted file mode 100644 index b4441c2fb..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1.toml +++ /dev/null @@ -1,30 +0,0 @@ -dataset = "homesite-insurance" -variant = "rla_only" -seed = 1 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1/report.json deleted file mode 100644 index 0beaadf5d..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1/report.json +++ /dev/null @@ -1,179 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 19, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "rla_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.9153453341450584, - "train_loss": 0.40274501984769645, - "val_score": 0.913644257591419 - }, - { - "epoch": 1, - "test_score": 0.9386762760531412, - "train_loss": 0.22882334332574497, - "val_score": 0.9339664964731247 - }, - { - "epoch": 2, - "test_score": 0.9417633747375609, - "train_loss": 0.20528600473295558, - "val_score": 0.9393599453707585 - }, - { - "epoch": 3, - "test_score": 0.9456237884409205, - "train_loss": 0.19763150689276782, - "val_score": 0.9433622452457857 - }, - { - "epoch": 4, - "test_score": 0.9483589387870844, - "train_loss": 0.19308107224377719, - "val_score": 0.9451832756433243 - }, - { - "epoch": 5, - "test_score": 0.9491328820263178, - "train_loss": 0.19030495123429733, - "val_score": 0.9464752936375359 - }, - { - "epoch": 6, - "test_score": 0.9487128442731776, - "train_loss": 0.18752631152218038, - "val_score": 0.9474417222531636 - }, - { - "epoch": 7, - "test_score": 0.9503851666943889, - "train_loss": 0.18584440404718572, - "val_score": 0.9485849338034206 - }, - { - "epoch": 8, - "test_score": 0.9505842158781422, - "train_loss": 0.18417474221099506, - "val_score": 0.9486520615552427 - }, - { - "epoch": 9, - "test_score": 0.9488525024768792, - "train_loss": 0.18312697573141618, - "val_score": 0.9484094608930247 - }, - { - "epoch": 10, - "test_score": 0.950910757537991, - "train_loss": 0.18155857798728076, - "val_score": 0.9492701907748727 - }, - { - "epoch": 11, - "test_score": 0.9498996007321764, - "train_loss": 0.1806687345558947, - "val_score": 0.9495692631528675 - }, - { - "epoch": 12, - "test_score": 0.9505066548342181, - "train_loss": 0.179966039955616, - "val_score": 0.9505498646481203 - }, - { - "epoch": 13, - "test_score": 0.9518296384476232, - "train_loss": 0.17865862548351288, - "val_score": 0.9497052682861931 - }, - { - "epoch": 14, - "test_score": 0.9514363967062314, - "train_loss": 0.1780030677264387, - "val_score": 0.9504259222882722 - }, - { - "epoch": 15, - "test_score": 0.951733279119533, - "train_loss": 0.1766317753629251, - "val_score": 0.9503818826145489 - }, - { - "epoch": 16, - "test_score": 0.951395369088604, - "train_loss": 0.17607535245743663, - "val_score": 0.9512202717475583 - }, - { - "epoch": 17, - "test_score": 0.9506180189623946, - "train_loss": 0.17567047043280168, - "val_score": 0.9505782191994796 - }, - { - "epoch": 18, - "test_score": 0.9511110631623314, - "train_loss": 0.17454163973981685, - "val_score": 0.9500785988163677 - }, - { - "epoch": 19, - "test_score": 0.9523188529200016, - "train_loss": 0.17404835318977183, - "val_score": 0.9514079656893926 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9523188529200016, - "score": 0.9523188529200016 - }, - "val": { - "roc-auc": 0.9514079656893926, - "score": 0.9514079656893926 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/1", - "seed": 1, - "task_type": "binclass", - "time_seconds": 58.812034130096436, - "variant": "rla_only" -} diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2.toml deleted file mode 100644 index f849c76fb..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2.toml +++ /dev/null @@ -1,30 +0,0 @@ -dataset = "homesite-insurance" -variant = "rla_only" -seed = 2 -batch_size = 2048 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/homesite-insurance" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2/report.json deleted file mode 100644 index fb427d9a5..000000000 --- a/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2/report.json +++ /dev/null @@ -1,179 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 19, - "config": { - "amp": true, - "batch_size": 2048, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/homesite-insurance" - }, - "dataset": "homesite-insurance", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "rla_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2.toml", - "dataset": "homesite-insurance", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": 0.9174934367892956, - "train_loss": 0.39997752498496664, - "val_score": 0.913728908016376 - }, - { - "epoch": 1, - "test_score": 0.9405841690032434, - "train_loss": 0.22608513818545775, - "val_score": 0.9356164412226329 - }, - { - "epoch": 2, - "test_score": 0.9455614254956344, - "train_loss": 0.20442303175275975, - "val_score": 0.9414721352482383 - }, - { - "epoch": 3, - "test_score": 0.9465339346722416, - "train_loss": 0.19598024392669852, - "val_score": 0.9439484766941502 - }, - { - "epoch": 4, - "test_score": 0.9478137643839494, - "train_loss": 0.19159091277555987, - "val_score": 0.9458952254575443 - }, - { - "epoch": 5, - "test_score": 0.949862221624715, - "train_loss": 0.18939599990844727, - "val_score": 0.9463206404411305 - }, - { - "epoch": 6, - "test_score": 0.9460805408377501, - "train_loss": 0.18695615828037263, - "val_score": 0.9467469346356111 - }, - { - "epoch": 7, - "test_score": 0.9487346145225781, - "train_loss": 0.1854052790186622, - "val_score": 0.9475296081742135 - }, - { - "epoch": 8, - "test_score": 0.9508778122160627, - "train_loss": 0.18390980579636312, - "val_score": 0.9494556427289249 - }, - { - "epoch": 9, - "test_score": 0.9500001763849286, - "train_loss": 0.1823697333986109, - "val_score": 0.9494444503742333 - }, - { - "epoch": 10, - "test_score": 0.9504842442833645, - "train_loss": 0.18122355111620642, - "val_score": 0.9495899158167878 - }, - { - "epoch": 11, - "test_score": 0.9515561934837319, - "train_loss": 0.18017809607765892, - "val_score": 0.9499209211344838 - }, - { - "epoch": 12, - "test_score": 0.9521558297538468, - "train_loss": 0.17910988574678247, - "val_score": 0.950046912055717 - }, - { - "epoch": 13, - "test_score": 0.9523240840620607, - "train_loss": 0.17836766649376262, - "val_score": 0.950473672231972 - }, - { - "epoch": 14, - "test_score": 0.9518764891841244, - "train_loss": 0.17786945118145508, - "val_score": 0.9510305468285569 - }, - { - "epoch": 15, - "test_score": 0.9507872035198894, - "train_loss": 0.1764437587423758, - "val_score": 0.9505283063969785 - }, - { - "epoch": 16, - "test_score": 0.9512406215166995, - "train_loss": 0.1760691683400761, - "val_score": 0.9499502252336094 - }, - { - "epoch": 17, - "test_score": 0.9515706546316413, - "train_loss": 0.17495807666670193, - "val_score": 0.9506721365072682 - }, - { - "epoch": 18, - "test_score": 0.95036785439284, - "train_loss": 0.17458564910021696, - "val_score": 0.9506224610917088 - }, - { - "epoch": 19, - "test_score": 0.9524476380801628, - "train_loss": 0.17387819655916908, - "val_score": 0.9513618950385021 - } - ], - "inference_mode": "mean", - "metric_direction": "higher", - "metrics": { - "test": { - "roc-auc": 0.9524476380801628, - "score": 0.9524476380801628 - }, - "val": { - "roc-auc": 0.9513618950385021, - "score": 0.9513618950385021 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/homesite-insurance/rla_only-evaluation/2", - "seed": 2, - "task_type": "binclass", - "time_seconds": 60.62201809883118, - "variant": "rla_only" -} diff --git a/tabm_integrated/paper/exp/integrated/manifest_baseline_fidelity.txt b/tabm_integrated/paper/exp/integrated/manifest_baseline_fidelity.txt new file mode 100644 index 000000000..fb976ae58 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/manifest_baseline_fidelity.txt @@ -0,0 +1,5 @@ +exp/integrated/baseline_fidelity/sberbank-housing/baseline_plr-evaluation/0.toml +exp/integrated/baseline_fidelity/ecom-offers/baseline_plr-evaluation/0.toml +exp/integrated/baseline_fidelity/homesite-insurance/baseline_plr-evaluation/0.toml +exp/integrated/baseline_fidelity/cooking-time/baseline_plr-evaluation/0.toml +exp/integrated/baseline_fidelity/delivery-eta/baseline_plr-evaluation/0.toml diff --git a/tabm_integrated/paper/exp/integrated/manifest_final.txt b/tabm_integrated/paper/exp/integrated/manifest_final.txt new file mode 100644 index 000000000..2a09cdc7e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/manifest_final.txt @@ -0,0 +1,90 @@ +exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml +exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml +exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml +exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml +exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml +exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml +exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml +exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml +exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml +exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml +exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml +exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml +exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml +exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml +exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml +exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml +exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml +exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml +exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml +exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml +exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml +exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml +exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml +exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml +exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml +exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml +exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml +exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml +exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml +exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml +exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml +exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml +exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml +exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml +exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml +exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml +exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml +exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml +exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml +exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml +exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml +exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml +exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml +exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml +exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml +exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml +exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml +exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml +exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml +exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml +exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml +exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml +exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml +exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml +exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml +exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml +exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml +exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml +exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml +exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml +exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml +exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml +exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml +exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml +exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml +exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml +exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml +exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml +exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml +exp/integrated/final/cooking-time/best_combined-evaluation/0.toml +exp/integrated/final/cooking-time/best_combined-evaluation/1.toml +exp/integrated/final/cooking-time/best_combined-evaluation/2.toml +exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml +exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml +exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml +exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml +exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml +exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml +exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml +exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml +exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml +exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml +exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml +exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml +exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml +exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml +exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml +exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml +exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml +exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml diff --git a/tabm_integrated/paper/exp/integrated/manifest_smoke.txt b/tabm_integrated/paper/exp/integrated/manifest_smoke.txt new file mode 100644 index 000000000..c28cfbdc2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/manifest_smoke.txt @@ -0,0 +1,30 @@ +exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0.toml +exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0.toml +exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0.toml +exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0.toml +exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0.toml +exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0.toml +exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0.toml +exp/integrated/smoke/ecom-offers/rla_only-evaluation/0.toml +exp/integrated/smoke/ecom-offers/esam_only-evaluation/0.toml +exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0.toml +exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0.toml +exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0.toml +exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0.toml +exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0.toml +exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0.toml +exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0.toml +exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0.toml +exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0.toml +exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0.toml +exp/integrated/smoke/cooking-time/rla_only-evaluation/0.toml +exp/integrated/smoke/cooking-time/esam_only-evaluation/0.toml +exp/integrated/smoke/cooking-time/mfb_only-evaluation/0.toml +exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0.toml +exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0.toml +exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0.toml +exp/integrated/smoke/delivery-eta/rla_only-evaluation/0.toml +exp/integrated/smoke/delivery-eta/esam_only-evaluation/0.toml +exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0.toml +exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0.toml +exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0.toml diff --git a/tabm_integrated/paper/exp/integrated/manifest_sweeps.txt b/tabm_integrated/paper/exp/integrated/manifest_sweeps.txt new file mode 100644 index 000000000..54d1eed59 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/manifest_sweeps.txt @@ -0,0 +1,155 @@ +exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0.toml +exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0.toml +exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0.toml +exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0.toml +exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0.toml +exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0.toml +exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0.toml +exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0.toml +exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0.toml +exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0.toml +exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0.toml +exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0.toml +exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0.toml +exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0.toml +exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0.toml +exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0.toml +exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0.toml +exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0.toml +exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0.toml +exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0.toml diff --git a/tabm_integrated/paper/exp/integrated/manifest_sweeps_remaining.txt b/tabm_integrated/paper/exp/integrated/manifest_sweeps_remaining.txt new file mode 100644 index 000000000..04872af88 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/manifest_sweeps_remaining.txt @@ -0,0 +1,18 @@ +exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0.toml +exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0.toml diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0.toml deleted file mode 100644 index 4e8978695..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0.toml +++ /dev/null @@ -1,35 +0,0 @@ -dataset = "sberbank-housing" -variant = "all_four_combined" -seed = 0 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0/report.json deleted file mode 100644 index 0d77d461d..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0/report.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_four_combined" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.529940564415122, - "train_loss": 0.6847074972955804, - "val_score": -11.629337281019204 - }, - { - "epoch": 1, - "test_score": -11.552026902735443, - "train_loss": 0.5718414093318739, - "val_score": -11.661667615617779 - }, - { - "epoch": 2, - "test_score": -11.542241675493882, - "train_loss": 0.5511937800206637, - "val_score": -11.652930392959707 - }, - { - "epoch": 3, - "test_score": -11.57101077716521, - "train_loss": 0.5341950419702028, - "val_score": -11.682380411755172 - }, - { - "epoch": 4, - "test_score": -11.572264141613829, - "train_loss": 0.5156774677728352, - "val_score": -11.690614010077367 - }, - { - "epoch": 5, - "test_score": -11.582711694809559, - "train_loss": 0.515990381178103, - "val_score": -11.692023559546577 - }, - { - "epoch": 6, - "test_score": -11.577212978586967, - "train_loss": 0.5116662116427171, - "val_score": -11.67999320252103 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.529940564415122, - "score": -11.529940564415122 - }, - "val": { - "rmse": 11.629337281019204, - "score": -11.629337281019204 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 6.405361175537109, - "variant": "all_four_combined" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1.toml deleted file mode 100644 index 5233af024..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1.toml +++ /dev/null @@ -1,35 +0,0 @@ -dataset = "sberbank-housing" -variant = "all_four_combined" -seed = 1 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1/report.json deleted file mode 100644 index 3a7d4d53d..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1/report.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_four_combined" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.5397613458401, - "train_loss": 0.6788979737382186, - "val_score": -11.640582398202245 - }, - { - "epoch": 1, - "test_score": -11.555400586515542, - "train_loss": 0.569147073908856, - "val_score": -11.668715451500475 - }, - { - "epoch": 2, - "test_score": -11.541570743387114, - "train_loss": 0.5481514789556202, - "val_score": -11.654005391915891 - }, - { - "epoch": 3, - "test_score": -11.571300230037759, - "train_loss": 0.533915389525263, - "val_score": -11.689820411157616 - }, - { - "epoch": 4, - "test_score": -11.551482028750549, - "train_loss": 0.5247134566307068, - "val_score": -11.675892987295715 - }, - { - "epoch": 5, - "test_score": -11.548386656801894, - "train_loss": 0.516610744752382, - "val_score": -11.66929604101179 - }, - { - "epoch": 6, - "test_score": -11.611992234214632, - "train_loss": 0.5065359049721768, - "val_score": -11.706363245247752 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.5397613458401, - "score": -11.5397613458401 - }, - "val": { - "rmse": 11.640582398202245, - "score": -11.640582398202245 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 6.378625154495239, - "variant": "all_four_combined" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2.toml deleted file mode 100644 index 645cf2440..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2.toml +++ /dev/null @@ -1,35 +0,0 @@ -dataset = "sberbank-housing" -variant = "all_four_combined" -seed = 2 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2/report.json deleted file mode 100644 index 5ba1ed4b9..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2/report.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_four_combined" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.537548960274977, - "train_loss": 0.6803103512839267, - "val_score": -11.640932383735219 - }, - { - "epoch": 1, - "test_score": -11.552889402224842, - "train_loss": 0.571699343229595, - "val_score": -11.667362592013484 - }, - { - "epoch": 2, - "test_score": -11.533435810787802, - "train_loss": 0.5532804304047635, - "val_score": -11.653257093143798 - }, - { - "epoch": 3, - "test_score": -11.540510392827223, - "train_loss": 0.5380085847879711, - "val_score": -11.664698262144418 - }, - { - "epoch": 4, - "test_score": -11.55832180104211, - "train_loss": 0.5288247748425132, - "val_score": -11.67843129218514 - }, - { - "epoch": 5, - "test_score": -11.562065157574914, - "train_loss": 0.5185170534409975, - "val_score": -11.677382062391155 - }, - { - "epoch": 6, - "test_score": -11.559098686799585, - "train_loss": 0.5019456665766867, - "val_score": -11.66983214531403 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.537548960274977, - "score": -11.537548960274977 - }, - "val": { - "rmse": 11.640932383735219, - "score": -11.640932383735219 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_four_combined-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 6.827687740325928, - "variant": "all_four_combined" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml deleted file mode 100644 index 1ee5d2edc..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml +++ /dev/null @@ -1,34 +0,0 @@ -dataset = "sberbank-housing" -variant = "all_minus_cf_fisd" -seed = 0 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0/report.json deleted file mode 100644 index 280a56631..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0/report.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 2, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_minus_cf_fisd" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.513746587172779, - "train_loss": 0.7153874667067277, - "val_score": -11.650224125224748 - }, - { - "epoch": 1, - "test_score": -11.52669511687469, - "train_loss": 0.563560238010005, - "val_score": -11.652123751275518 - }, - { - "epoch": 2, - "test_score": -11.495833595850131, - "train_loss": 0.537884428312904, - "val_score": -11.627223960865845 - }, - { - "epoch": 3, - "test_score": -11.543432731791633, - "train_loss": 0.5185733155200356, - "val_score": -11.665168520694888 - }, - { - "epoch": 4, - "test_score": -11.594056652363017, - "train_loss": 0.5044406621079696, - "val_score": -11.705019299712436 - }, - { - "epoch": 5, - "test_score": -11.58571953288731, - "train_loss": 0.5075031064058605, - "val_score": -11.69188522239724 - }, - { - "epoch": 6, - "test_score": -11.565673392647339, - "train_loss": 0.5029034708675585, - "val_score": -11.668180603617378 - }, - { - "epoch": 7, - "test_score": -11.585750483150088, - "train_loss": 0.4975504420305553, - "val_score": -11.693598008098277 - }, - { - "epoch": 8, - "test_score": -11.576650837115723, - "train_loss": 0.48497160328061956, - "val_score": -11.689399442493693 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.495833595850131, - "score": -11.495833595850131 - }, - "val": { - "rmse": 11.627223960865845, - "score": -11.627223960865845 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 8.408822298049927, - "variant": "all_minus_cf_fisd" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1.toml deleted file mode 100644 index afca58636..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1.toml +++ /dev/null @@ -1,34 +0,0 @@ -dataset = "sberbank-housing" -variant = "all_minus_cf_fisd" -seed = 1 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1/report.json deleted file mode 100644 index e873837eb..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1/report.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_minus_cf_fisd" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.52660642338622, - "train_loss": 0.7090223055136832, - "val_score": -11.641336754666401 - }, - { - "epoch": 1, - "test_score": -11.571683958219143, - "train_loss": 0.5640872654161955, - "val_score": -11.688843349018354 - }, - { - "epoch": 2, - "test_score": -11.584105395011493, - "train_loss": 0.5382686147564336, - "val_score": -11.7023198623663 - }, - { - "epoch": 3, - "test_score": -11.580835599119268, - "train_loss": 0.526674791386253, - "val_score": -11.693449250374918 - }, - { - "epoch": 4, - "test_score": -11.584639514489178, - "train_loss": 0.5151608900019997, - "val_score": -11.702422218968842 - }, - { - "epoch": 5, - "test_score": -11.582740677050353, - "train_loss": 0.5098892026825955, - "val_score": -11.69189435791984 - }, - { - "epoch": 6, - "test_score": -11.622656144972105, - "train_loss": 0.49971666304688706, - "val_score": -11.708356719259525 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.52660642338622, - "score": -11.52660642338622 - }, - "val": { - "rmse": 11.641336754666401, - "score": -11.641336754666401 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 6.753610610961914, - "variant": "all_minus_cf_fisd" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2.toml deleted file mode 100644 index 1ee1df5df..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2.toml +++ /dev/null @@ -1,34 +0,0 @@ -dataset = "sberbank-housing" -variant = "all_minus_cf_fisd" -seed = 2 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2/report.json deleted file mode 100644 index 745a80212..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2/report.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_minus_cf_fisd" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.512635297114862, - "train_loss": 0.7103617254056429, - "val_score": -11.642066159740436 - }, - { - "epoch": 1, - "test_score": -11.561014605812948, - "train_loss": 0.5649806794367338, - "val_score": -11.6840142748627 - }, - { - "epoch": 2, - "test_score": -11.550686136317902, - "train_loss": 0.5418291295829573, - "val_score": -11.675974012502126 - }, - { - "epoch": 3, - "test_score": -11.552818080053902, - "train_loss": 0.523169795149251, - "val_score": -11.678895771660628 - }, - { - "epoch": 4, - "test_score": -11.597623357986365, - "train_loss": 0.5167672179247204, - "val_score": -11.710011064506274 - }, - { - "epoch": 5, - "test_score": -11.571608136649118, - "train_loss": 0.5108516859380823, - "val_score": -11.677881209615828 - }, - { - "epoch": 6, - "test_score": -11.561509538447702, - "train_loss": 0.4943600296974182, - "val_score": -11.662550790559619 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.512635297114862, - "score": -11.512635297114862 - }, - "val": { - "rmse": 11.642066159740436, - "score": -11.642066159740436 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_cf_fisd-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 7.154858112335205, - "variant": "all_minus_cf_fisd" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0.toml deleted file mode 100644 index 35a1f75ee..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "sberbank-housing" -variant = "all_minus_esam" -seed = 0 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0/report.json deleted file mode 100644 index 213ca4f77..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0/report.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_minus_esam" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.529550814849099, - "train_loss": 0.6817392173566317, - "val_score": -11.629953949067119 - }, - { - "epoch": 1, - "test_score": -11.550435138118512, - "train_loss": 0.5682083619268317, - "val_score": -11.660886440704575 - }, - { - "epoch": 2, - "test_score": -11.530798761072859, - "train_loss": 0.5492272831891712, - "val_score": -11.642030771869832 - }, - { - "epoch": 3, - "test_score": -11.537696421760417, - "train_loss": 0.534926375276164, - "val_score": -11.651297409440813 - }, - { - "epoch": 4, - "test_score": -11.564843512505908, - "train_loss": 0.5191069511990798, - "val_score": -11.687417096338654 - }, - { - "epoch": 5, - "test_score": -11.562500659839507, - "train_loss": 0.5150835153303648, - "val_score": -11.682755920101723 - }, - { - "epoch": 6, - "test_score": -11.584154131965139, - "train_loss": 0.5153278159467798, - "val_score": -11.693979680442478 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.529550814849099, - "score": -11.529550814849099 - }, - "val": { - "rmse": 11.629953949067119, - "score": -11.629953949067119 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 5.24409818649292, - "variant": "all_minus_esam" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1.toml deleted file mode 100644 index 5e5dbfdac..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "sberbank-housing" -variant = "all_minus_esam" -seed = 1 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1/report.json deleted file mode 100644 index 88e5e3b87..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1/report.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_minus_esam" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.535907580086148, - "train_loss": 0.6764907836914062, - "val_score": -11.636771200368536 - }, - { - "epoch": 1, - "test_score": -11.55272892672103, - "train_loss": 0.5704263276175449, - "val_score": -11.669556904596222 - }, - { - "epoch": 2, - "test_score": -11.540304789889854, - "train_loss": 0.5479874469731983, - "val_score": -11.660911957294818 - }, - { - "epoch": 3, - "test_score": -11.566767714294487, - "train_loss": 0.5281973810572373, - "val_score": -11.694827144237479 - }, - { - "epoch": 4, - "test_score": -11.54475385414234, - "train_loss": 0.5193592780514767, - "val_score": -11.669544482653006 - }, - { - "epoch": 5, - "test_score": -11.54118006445933, - "train_loss": 0.5155376776268608, - "val_score": -11.653920940654078 - }, - { - "epoch": 6, - "test_score": -11.589382284586234, - "train_loss": 0.5071287123780501, - "val_score": -11.688474563520696 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.535907580086148, - "score": -11.535907580086148 - }, - "val": { - "rmse": 11.636771200368536, - "score": -11.636771200368536 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 5.403448104858398, - "variant": "all_minus_esam" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2.toml deleted file mode 100644 index fcafe5ea2..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "sberbank-housing" -variant = "all_minus_esam" -seed = 2 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2/report.json deleted file mode 100644 index 6f5cfb23c..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2/report.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_minus_esam" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.53778767500918, - "train_loss": 0.6787219533794805, - "val_score": -11.640474254476947 - }, - { - "epoch": 1, - "test_score": -11.552421177337404, - "train_loss": 0.5701834920205568, - "val_score": -11.668965866960828 - }, - { - "epoch": 2, - "test_score": -11.534591397446842, - "train_loss": 0.5533530414104462, - "val_score": -11.655044943364052 - }, - { - "epoch": 3, - "test_score": -11.539088285981682, - "train_loss": 0.5404668748378754, - "val_score": -11.660349270806064 - }, - { - "epoch": 4, - "test_score": -11.551949631261518, - "train_loss": 0.5275069977107801, - "val_score": -11.672618837158101 - }, - { - "epoch": 5, - "test_score": -11.547939390852582, - "train_loss": 0.5141889217652773, - "val_score": -11.66554000500613 - }, - { - "epoch": 6, - "test_score": -11.560263587560069, - "train_loss": 0.5009887877263521, - "val_score": -11.672157376102328 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.53778767500918, - "score": -11.53778767500918 - }, - "val": { - "rmse": 11.640474254476947, - "score": -11.640474254476947 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_esam-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 5.441632270812988, - "variant": "all_minus_esam" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0.toml deleted file mode 100644 index 67db20514..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "sberbank-housing" -variant = "all_minus_mfb" -seed = 0 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0/report.json deleted file mode 100644 index 060d2806a..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0/report.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_minus_mfb" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.529854542723736, - "train_loss": 0.6731284982279727, - "val_score": -11.628387941925986 - }, - { - "epoch": 1, - "test_score": -11.55628990220092, - "train_loss": 0.5665753781795502, - "val_score": -11.665185525516796 - }, - { - "epoch": 2, - "test_score": -11.533286971805715, - "train_loss": 0.5451070242806485, - "val_score": -11.645597852442798 - }, - { - "epoch": 3, - "test_score": -11.543267498249048, - "train_loss": 0.5240256049131092, - "val_score": -11.662766667595363 - }, - { - "epoch": 4, - "test_score": -11.569713754764527, - "train_loss": 0.5067465054361444, - "val_score": -11.69366259967839 - }, - { - "epoch": 5, - "test_score": -11.55628462064683, - "train_loss": 0.5012774718435187, - "val_score": -11.671706355458351 - }, - { - "epoch": 6, - "test_score": -11.601807788863189, - "train_loss": 0.49037294011366994, - "val_score": -11.707959224566062 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.529854542723736, - "score": -11.529854542723736 - }, - "val": { - "rmse": 11.628387941925986, - "score": -11.628387941925986 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 6.601038455963135, - "variant": "all_minus_mfb" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1.toml deleted file mode 100644 index 75c9b75bd..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "sberbank-housing" -variant = "all_minus_mfb" -seed = 1 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1/report.json deleted file mode 100644 index f6828bf47..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1/report.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_minus_mfb" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.527989037555509, - "train_loss": 0.6697249318424024, - "val_score": -11.635427085077215 - }, - { - "epoch": 1, - "test_score": -11.541932985467193, - "train_loss": 0.5654332637786865, - "val_score": -11.66612860029216 - }, - { - "epoch": 2, - "test_score": -11.535246200546677, - "train_loss": 0.5421643006174188, - "val_score": -11.660955793357955 - }, - { - "epoch": 3, - "test_score": -11.5517871613484, - "train_loss": 0.5226815766409824, - "val_score": -11.682632493472482 - }, - { - "epoch": 4, - "test_score": -11.538862160794725, - "train_loss": 0.5097503881705435, - "val_score": -11.677146854732632 - }, - { - "epoch": 5, - "test_score": -11.533347169136466, - "train_loss": 0.5043469306669737, - "val_score": -11.656677401891065 - }, - { - "epoch": 6, - "test_score": -11.604282088650239, - "train_loss": 0.4942637085914612, - "val_score": -11.70735774382063 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.527989037555509, - "score": -11.527989037555509 - }, - "val": { - "rmse": 11.635427085077215, - "score": -11.635427085077215 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 6.853307008743286, - "variant": "all_minus_mfb" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2.toml deleted file mode 100644 index 039ada680..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2.toml +++ /dev/null @@ -1,33 +0,0 @@ -dataset = "sberbank-housing" -variant = "all_minus_mfb" -seed = 2 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 -esam_enabled = true -esam_rho = 0.0025 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2/report.json deleted file mode 100644 index e9bbf8124..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2/report.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_minus_mfb" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.53653783950218, - "train_loss": 0.6675687244063929, - "val_score": -11.638006993773004 - }, - { - "epoch": 1, - "test_score": -11.563427037372303, - "train_loss": 0.5652898048099718, - "val_score": -11.677691091918867 - }, - { - "epoch": 2, - "test_score": -11.543884137773045, - "train_loss": 0.5458332911918038, - "val_score": -11.662630600111127 - }, - { - "epoch": 3, - "test_score": -11.554151335599578, - "train_loss": 0.5292629395660601, - "val_score": -11.67665486339423 - }, - { - "epoch": 4, - "test_score": -11.5687773289453, - "train_loss": 0.5131576892576719, - "val_score": -11.6912705158342 - }, - { - "epoch": 5, - "test_score": -11.555286363579915, - "train_loss": 0.502765333966205, - "val_score": -11.674081535071613 - }, - { - "epoch": 6, - "test_score": -11.562772510513334, - "train_loss": 0.4887320367913497, - "val_score": -11.675086625163777 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.53653783950218, - "score": -11.53653783950218 - }, - "val": { - "rmse": 11.638006993773004, - "score": -11.638006993773004 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_mfb-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 7.148878574371338, - "variant": "all_minus_mfb" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0.toml deleted file mode 100644 index dbdd34242..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0.toml +++ /dev/null @@ -1,32 +0,0 @@ -dataset = "sberbank-housing" -variant = "all_minus_rla" -seed = 0 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0/report.json deleted file mode 100644 index fa2c9ee04..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0/report.json +++ /dev/null @@ -1,127 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 4, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "all_minus_rla" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.566356778642936, - "train_loss": 0.6788321796216463, - "val_score": -11.664222097869944 - }, - { - "epoch": 1, - "test_score": -11.559857700620443, - "train_loss": 0.5704584325614729, - "val_score": -11.670672864172788 - }, - { - "epoch": 2, - "test_score": -11.561442228854185, - "train_loss": 0.5435581897434435, - "val_score": -11.671773029173043 - }, - { - "epoch": 3, - "test_score": -11.562635926143608, - "train_loss": 0.5292612361280542, - "val_score": -11.67968292746566 - }, - { - "epoch": 4, - "test_score": -11.530414333922007, - "train_loss": 0.516992837190628, - "val_score": -11.648797073801292 - }, - { - "epoch": 5, - "test_score": -11.546760692727513, - "train_loss": 0.5125646606871956, - "val_score": -11.659984818672779 - }, - { - "epoch": 6, - "test_score": -11.536368539200957, - "train_loss": 0.5112680792808533, - "val_score": -11.652399403780938 - }, - { - "epoch": 7, - "test_score": -11.557969974086063, - "train_loss": 0.5115332070149874, - "val_score": -11.681961787352837 - }, - { - "epoch": 8, - "test_score": -11.566561258881947, - "train_loss": 0.4950885882503108, - "val_score": -11.689068530901732 - }, - { - "epoch": 9, - "test_score": -11.561465985226054, - "train_loss": 0.4844362704377425, - "val_score": -11.683070031725862 - }, - { - "epoch": 10, - "test_score": -11.546094648656844, - "train_loss": 0.4920222649448796, - "val_score": -11.659160997097947 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.530414333922007, - "score": -11.530414333922007 - }, - "val": { - "rmse": 11.648797073801292, - "score": -11.648797073801292 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 9.661595106124878, - "variant": "all_minus_rla" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1.toml deleted file mode 100644 index 9be092dff..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1.toml +++ /dev/null @@ -1,32 +0,0 @@ -dataset = "sberbank-housing" -variant = "all_minus_rla" -seed = 1 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1/report.json deleted file mode 100644 index 916dc5a03..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1/report.json +++ /dev/null @@ -1,133 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 5, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "all_minus_rla" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.56395749365411, - "train_loss": 0.6744678585152877, - "val_score": -11.668505569129103 - }, - { - "epoch": 1, - "test_score": -11.537794287535394, - "train_loss": 0.5657629998106706, - "val_score": -11.657687264774315 - }, - { - "epoch": 2, - "test_score": -11.548418367762531, - "train_loss": 0.5490596780651494, - "val_score": -11.67048524398528 - }, - { - "epoch": 3, - "test_score": -11.549676165644213, - "train_loss": 0.536047350419195, - "val_score": -11.674848103811472 - }, - { - "epoch": 4, - "test_score": -11.556204736846864, - "train_loss": 0.52565745617214, - "val_score": -11.684645686721474 - }, - { - "epoch": 5, - "test_score": -11.537306272919873, - "train_loss": 0.5129967410313455, - "val_score": -11.650615731532309 - }, - { - "epoch": 6, - "test_score": -11.59410995377037, - "train_loss": 0.49969515361283956, - "val_score": -11.707428776200379 - }, - { - "epoch": 7, - "test_score": -11.594070471269934, - "train_loss": 0.4978504071110173, - "val_score": -11.699758047878865 - }, - { - "epoch": 8, - "test_score": -11.604694968524592, - "train_loss": 0.49357654075873525, - "val_score": -11.712956901440387 - }, - { - "epoch": 9, - "test_score": -11.56541084559229, - "train_loss": 0.48850819939061213, - "val_score": -11.66822833553725 - }, - { - "epoch": 10, - "test_score": -11.583416469650697, - "train_loss": 0.49180947636303146, - "val_score": -11.685402421857033 - }, - { - "epoch": 11, - "test_score": -11.57456217322941, - "train_loss": 0.49050161556193705, - "val_score": -11.673448898234296 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.537306272919873, - "score": -11.537306272919873 - }, - "val": { - "rmse": 11.650615731532309, - "score": -11.650615731532309 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 10.637221097946167, - "variant": "all_minus_rla" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2.toml deleted file mode 100644 index e24f8e419..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2.toml +++ /dev/null @@ -1,32 +0,0 @@ -dataset = "sberbank-housing" -variant = "all_minus_rla" -seed = 2 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 -mfb_enabled = true -mfb_keep = 0.8 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2/report.json deleted file mode 100644 index bc7bfb855..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2/report.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "all_minus_rla" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.537770482423289, - "train_loss": 0.6824794066579718, - "val_score": -11.64186693773376 - }, - { - "epoch": 1, - "test_score": -11.56087866060679, - "train_loss": 0.5675985232779854, - "val_score": -11.674113558132115 - }, - { - "epoch": 2, - "test_score": -11.552444952260107, - "train_loss": 0.562320916276229, - "val_score": -11.668439530592837 - }, - { - "epoch": 3, - "test_score": -11.552125967971559, - "train_loss": 0.5406928344776756, - "val_score": -11.67393056803177 - }, - { - "epoch": 4, - "test_score": -11.531967843721764, - "train_loss": 0.5256602340622952, - "val_score": -11.65659100644788 - }, - { - "epoch": 5, - "test_score": -11.576962555511646, - "train_loss": 0.5158363957154123, - "val_score": -11.694865634206682 - }, - { - "epoch": 6, - "test_score": -11.559399658242826, - "train_loss": 0.5086805240104073, - "val_score": -11.677255312186018 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.537770482423289, - "score": -11.537770482423289 - }, - "val": { - "rmse": 11.64186693773376, - "score": -11.64186693773376 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/all_minus_rla-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 6.342571973800659, - "variant": "all_minus_rla" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0.toml deleted file mode 100644 index 54fe49077..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0.toml +++ /dev/null @@ -1,27 +0,0 @@ -dataset = "sberbank-housing" -variant = "baseline_plr" -seed = 0 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0/report.json deleted file mode 100644 index 18b204a91..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0/report.json +++ /dev/null @@ -1,128 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 5, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "baseline_plr" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.591843440565695, - "train_loss": 0.693355569713994, - "val_score": -11.707670543731014 - }, - { - "epoch": 1, - "test_score": -11.569860146879883, - "train_loss": 0.5572547583203566, - "val_score": -11.699261138278803 - }, - { - "epoch": 2, - "test_score": -11.557012792142023, - "train_loss": 0.5240200240361063, - "val_score": -11.679103507557178 - }, - { - "epoch": 3, - "test_score": -11.546682725268232, - "train_loss": 0.5031786121820149, - "val_score": -11.66853172290153 - }, - { - "epoch": 4, - "test_score": -11.576095918058972, - "train_loss": 0.4946268655751881, - "val_score": -11.689467973309283 - }, - { - "epoch": 5, - "test_score": -11.546495072614636, - "train_loss": 0.49316220534475225, - "val_score": -11.653776913834497 - }, - { - "epoch": 6, - "test_score": -11.576089327411522, - "train_loss": 0.4814813748786324, - "val_score": -11.692044440483546 - }, - { - "epoch": 7, - "test_score": -11.570189192666712, - "train_loss": 0.47625335894132914, - "val_score": -11.6955636506517 - }, - { - "epoch": 8, - "test_score": -11.549253391724072, - "train_loss": 0.4664542518163982, - "val_score": -11.665307174673636 - }, - { - "epoch": 9, - "test_score": -11.559674221847015, - "train_loss": 0.46214914792462397, - "val_score": -11.676656823560215 - }, - { - "epoch": 10, - "test_score": -11.541870849859816, - "train_loss": 0.46384631332598236, - "val_score": -11.672688120104073 - }, - { - "epoch": 11, - "test_score": -11.534430667367701, - "train_loss": 0.45383137621377645, - "val_score": -11.660644357455324 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.546495072614636, - "score": -11.546495072614636 - }, - "val": { - "rmse": 11.653776913834497, - "score": -11.653776913834497 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 8.12721848487854, - "variant": "baseline_plr" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1.toml deleted file mode 100644 index 96b89a263..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1.toml +++ /dev/null @@ -1,27 +0,0 @@ -dataset = "sberbank-housing" -variant = "baseline_plr" -seed = 1 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1/report.json deleted file mode 100644 index 8fd14007c..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1/report.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 1, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "baseline_plr" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.5648421930942, - "train_loss": 0.689470742878161, - "val_score": -11.693471433664252 - }, - { - "epoch": 1, - "test_score": -11.510137982108967, - "train_loss": 0.5526794289287768, - "val_score": -11.654311113459125 - }, - { - "epoch": 2, - "test_score": -11.529714259819535, - "train_loss": 0.5296151810570767, - "val_score": -11.669904059871339 - }, - { - "epoch": 3, - "test_score": -11.544838443119694, - "train_loss": 0.5117689402479875, - "val_score": -11.687559403049239 - }, - { - "epoch": 4, - "test_score": -11.559208251872455, - "train_loss": 0.5005878413978376, - "val_score": -11.688715417668204 - }, - { - "epoch": 5, - "test_score": -11.55868681844079, - "train_loss": 0.48899377960907786, - "val_score": -11.673540397418268 - }, - { - "epoch": 6, - "test_score": -11.59862849579201, - "train_loss": 0.47632234974911336, - "val_score": -11.72544405163437 - }, - { - "epoch": 7, - "test_score": -11.569045075332628, - "train_loss": 0.47122232223811905, - "val_score": -11.684690739624394 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.510137982108967, - "score": -11.510137982108967 - }, - "val": { - "rmse": 11.654311113459125, - "score": -11.654311113459125 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 5.4021241664886475, - "variant": "baseline_plr" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2.toml deleted file mode 100644 index eb19da287..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2.toml +++ /dev/null @@ -1,27 +0,0 @@ -dataset = "sberbank-housing" -variant = "baseline_plr" -seed = 2 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2/report.json deleted file mode 100644 index df5e73c37..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2/report.json +++ /dev/null @@ -1,176 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 18, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "baseline_plr" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.554990567146126, - "train_loss": 0.6944742610580042, - "val_score": -11.681583641026789 - }, - { - "epoch": 1, - "test_score": -11.558377247465431, - "train_loss": 0.5524072521611264, - "val_score": -11.694212592316632 - }, - { - "epoch": 2, - "test_score": -11.554818896100436, - "train_loss": 0.5426086312846133, - "val_score": -11.695455362980141 - }, - { - "epoch": 3, - "test_score": -11.546551897331664, - "train_loss": 0.5175957852288297, - "val_score": -11.6712468193992 - }, - { - "epoch": 4, - "test_score": -11.541549590213585, - "train_loss": 0.500424231353559, - "val_score": -11.670191059918428 - }, - { - "epoch": 5, - "test_score": -11.548839851010875, - "train_loss": 0.4906174760115774, - "val_score": -11.668238797301818 - }, - { - "epoch": 6, - "test_score": -11.580356645379265, - "train_loss": 0.4788107119108501, - "val_score": -11.70130407124665 - }, - { - "epoch": 7, - "test_score": -11.558303978920982, - "train_loss": 0.47477054439092936, - "val_score": -11.685612000748238 - }, - { - "epoch": 8, - "test_score": -11.548932998054612, - "train_loss": 0.46741848870327596, - "val_score": -11.67610796423118 - }, - { - "epoch": 9, - "test_score": -11.55564223362945, - "train_loss": 0.4634051119026385, - "val_score": -11.686815864000874 - }, - { - "epoch": 10, - "test_score": -11.528746790674022, - "train_loss": 0.45700703482878835, - "val_score": -11.662469017911688 - }, - { - "epoch": 11, - "test_score": -11.520487572630458, - "train_loss": 0.45120432502345037, - "val_score": -11.655515591391566 - }, - { - "epoch": 12, - "test_score": -11.500814657795715, - "train_loss": 0.454963019019679, - "val_score": -11.632714121897644 - }, - { - "epoch": 13, - "test_score": -11.525585075306372, - "train_loss": 0.4519016460368508, - "val_score": -11.661778833921053 - }, - { - "epoch": 14, - "test_score": -11.52689632942267, - "train_loss": 0.44427210406253215, - "val_score": -11.68127928638818 - }, - { - "epoch": 15, - "test_score": -11.49810377326885, - "train_loss": 0.44250490634064926, - "val_score": -11.632058902512634 - }, - { - "epoch": 16, - "test_score": -11.530547329962946, - "train_loss": 0.43535852589105306, - "val_score": -11.6786279306174 - }, - { - "epoch": 17, - "test_score": -11.479069113388983, - "train_loss": 0.4340797850960179, - "val_score": -11.643101535007421 - }, - { - "epoch": 18, - "test_score": -11.460434895463788, - "train_loss": 0.43446698314265203, - "val_score": -11.624637064857055 - }, - { - "epoch": 19, - "test_score": -11.504066080171528, - "train_loss": 0.4255804733226174, - "val_score": -11.66930388661818 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.460434895463788, - "score": -11.460434895463788 - }, - "val": { - "rmse": 11.624637064857055, - "score": -11.624637064857055 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/baseline_plr-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 14.094439268112183, - "variant": "baseline_plr" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0.toml deleted file mode 100644 index d24995252..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0.toml +++ /dev/null @@ -1,28 +0,0 @@ -dataset = "sberbank-housing" -variant = "cf_fisd_only" -seed = 0 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0/report.json deleted file mode 100644 index ce44d72ea..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0/report.json +++ /dev/null @@ -1,129 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 5, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "cf_fisd_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.567909418746003, - "train_loss": 0.6687586307525635, - "val_score": -11.665182909391962 - }, - { - "epoch": 1, - "test_score": -11.56754864967921, - "train_loss": 0.5649684698958146, - "val_score": -11.678224850932839 - }, - { - "epoch": 2, - "test_score": -11.555461989129928, - "train_loss": 0.5369522273540497, - "val_score": -11.666910078143415 - }, - { - "epoch": 3, - "test_score": -11.552720341548957, - "train_loss": 0.5171568503505305, - "val_score": -11.671635759345309 - }, - { - "epoch": 4, - "test_score": -11.56288138095683, - "train_loss": 0.5048333422133797, - "val_score": -11.681953950247498 - }, - { - "epoch": 5, - "test_score": -11.541630897512452, - "train_loss": 0.5006356631454668, - "val_score": -11.652210179841218 - }, - { - "epoch": 6, - "test_score": -11.572904287492442, - "train_loss": 0.4888252600243217, - "val_score": -11.679139436346876 - }, - { - "epoch": 7, - "test_score": -11.592184367207084, - "train_loss": 0.48512670554612813, - "val_score": -11.697678320956939 - }, - { - "epoch": 8, - "test_score": -11.589911552927722, - "train_loss": 0.473800927400589, - "val_score": -11.688119474099343 - }, - { - "epoch": 9, - "test_score": -11.600638510132683, - "train_loss": 0.46804546839312505, - "val_score": -11.697391995320475 - }, - { - "epoch": 10, - "test_score": -11.593598644985759, - "train_loss": 0.4702984408328408, - "val_score": -11.684281992433721 - }, - { - "epoch": 11, - "test_score": -11.600327427945595, - "train_loss": 0.4600174160380113, - "val_score": -11.683457272063137 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.541630897512452, - "score": -11.541630897512452 - }, - "val": { - "rmse": 11.652210179841218, - "score": -11.652210179841218 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 7.491765737533569, - "variant": "cf_fisd_only" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1.toml deleted file mode 100644 index 04d71bf4b..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1.toml +++ /dev/null @@ -1,28 +0,0 @@ -dataset = "sberbank-housing" -variant = "cf_fisd_only" -seed = 1 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1/report.json deleted file mode 100644 index 17c094d35..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1/report.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 1, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "cf_fisd_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.552516937144382, - "train_loss": 0.6644222955954703, - "val_score": -11.662240051447155 - }, - { - "epoch": 1, - "test_score": -11.531481567373374, - "train_loss": 0.5625829837824169, - "val_score": -11.653860711473339 - }, - { - "epoch": 2, - "test_score": -11.545619540172206, - "train_loss": 0.5438563996239713, - "val_score": -11.6724397453735 - }, - { - "epoch": 3, - "test_score": -11.532994580301242, - "train_loss": 0.5255994828123796, - "val_score": -11.674658590221908 - }, - { - "epoch": 4, - "test_score": -11.526772557646096, - "train_loss": 0.5107194577392778, - "val_score": -11.666102441132265 - }, - { - "epoch": 5, - "test_score": -11.546941733780434, - "train_loss": 0.4957810671705949, - "val_score": -11.657945770373138 - }, - { - "epoch": 6, - "test_score": -11.59458044321988, - "train_loss": 0.4840890231885408, - "val_score": -11.704705777126632 - }, - { - "epoch": 7, - "test_score": -11.591498554764918, - "train_loss": 0.47806531661435175, - "val_score": -11.689960730676798 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.531481567373374, - "score": -11.531481567373374 - }, - "val": { - "rmse": 11.653860711473339, - "score": -11.653860711473339 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 5.391960620880127, - "variant": "cf_fisd_only" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2.toml deleted file mode 100644 index 7bb90cee0..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2.toml +++ /dev/null @@ -1,28 +0,0 @@ -dataset = "sberbank-housing" -variant = "cf_fisd_only" -seed = 2 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -cf_fisd_enabled = true diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2/report.json deleted file mode 100644 index f7c307400..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2/report.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "cf_fisd_enabled": true, - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "cf_fisd_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.533345184614086, - "train_loss": 0.6680415868759155, - "val_score": -11.634575294575916 - }, - { - "epoch": 1, - "test_score": -11.570453609453505, - "train_loss": 0.5608573979453036, - "val_score": -11.68285126464747 - }, - { - "epoch": 2, - "test_score": -11.558421472398113, - "train_loss": 0.5553898183923018, - "val_score": -11.674624608149546 - }, - { - "epoch": 3, - "test_score": -11.556772494019064, - "train_loss": 0.5318876630381534, - "val_score": -11.678057604710867 - }, - { - "epoch": 4, - "test_score": -11.528811644085643, - "train_loss": 0.512807066503324, - "val_score": -11.649647824444898 - }, - { - "epoch": 5, - "test_score": -11.562042062299835, - "train_loss": 0.5008556623207895, - "val_score": -11.673831229347504 - }, - { - "epoch": 6, - "test_score": -11.598899499715143, - "train_loss": 0.48822733289317083, - "val_score": -11.710128990394585 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.533345184614086, - "score": -11.533345184614086 - }, - "val": { - "rmse": 11.634575294575916, - "score": -11.634575294575916 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/cf_fisd_only-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 5.544011116027832, - "variant": "cf_fisd_only" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0.toml deleted file mode 100644 index ab2fd5f22..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "sberbank-housing" -variant = "esam_only" -seed = 0 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0/report.json deleted file mode 100644 index a205243b3..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0/report.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 5, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "esam_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.591146418407833, - "train_loss": 0.6944636043749357, - "val_score": -11.706608293368964 - }, - { - "epoch": 1, - "test_score": -11.570934291825452, - "train_loss": 0.5573361547369706, - "val_score": -11.699037456859099 - }, - { - "epoch": 2, - "test_score": -11.554540255358294, - "train_loss": 0.524962704432638, - "val_score": -11.675947875400203 - }, - { - "epoch": 3, - "test_score": -11.547431983986945, - "train_loss": 0.5038708526837198, - "val_score": -11.668148564273972 - }, - { - "epoch": 4, - "test_score": -11.572368307825029, - "train_loss": 0.4954853465682582, - "val_score": -11.68653384208455 - }, - { - "epoch": 5, - "test_score": -11.547765632938749, - "train_loss": 0.49385242399416474, - "val_score": -11.655595449115308 - }, - { - "epoch": 6, - "test_score": -11.570127868209529, - "train_loss": 0.48245343722795186, - "val_score": -11.685794155049098 - }, - { - "epoch": 7, - "test_score": -11.57227996438985, - "train_loss": 0.4776828602740639, - "val_score": -11.697226327628513 - }, - { - "epoch": 8, - "test_score": -11.553689104939329, - "train_loss": 0.46720715102396515, - "val_score": -11.666946698415652 - }, - { - "epoch": 9, - "test_score": -11.565484728783257, - "train_loss": 0.46342792165906804, - "val_score": -11.678648182228018 - }, - { - "epoch": 10, - "test_score": -11.55179376586097, - "train_loss": 0.4648975808369486, - "val_score": -11.678585467448864 - }, - { - "epoch": 11, - "test_score": -11.540711364505574, - "train_loss": 0.45520147994944926, - "val_score": -11.660473587599665 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.547765632938749, - "score": -11.547765632938749 - }, - "val": { - "rmse": 11.655595449115308, - "score": -11.655595449115308 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 9.331769227981567, - "variant": "esam_only" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1.toml deleted file mode 100644 index 673db2d27..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "sberbank-housing" -variant = "esam_only" -seed = 1 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1/report.json deleted file mode 100644 index e251d10ea..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1/report.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 1, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "esam_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.566685924092585, - "train_loss": 0.6905955389926308, - "val_score": -11.69451269601541 - }, - { - "epoch": 1, - "test_score": -11.510244699059466, - "train_loss": 0.5533220720918555, - "val_score": -11.65383845278429 - }, - { - "epoch": 2, - "test_score": -11.530006734510243, - "train_loss": 0.5304650350620872, - "val_score": -11.670209364921169 - }, - { - "epoch": 3, - "test_score": -11.544808704877989, - "train_loss": 0.5126697138736123, - "val_score": -11.687711499590339 - }, - { - "epoch": 4, - "test_score": -11.562260475770744, - "train_loss": 0.5019068388562453, - "val_score": -11.690407783960037 - }, - { - "epoch": 5, - "test_score": -11.558812888679695, - "train_loss": 0.49016095619452626, - "val_score": -11.672905116858763 - }, - { - "epoch": 6, - "test_score": -11.592663490404515, - "train_loss": 0.4767398347980098, - "val_score": -11.721967657216767 - }, - { - "epoch": 7, - "test_score": -11.5657268248608, - "train_loss": 0.4714427041380029, - "val_score": -11.684485061914769 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.510244699059466, - "score": -11.510244699059466 - }, - "val": { - "rmse": 11.65383845278429, - "score": -11.65383845278429 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 6.739201784133911, - "variant": "esam_only" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2.toml deleted file mode 100644 index ba0118965..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "sberbank-housing" -variant = "esam_only" -seed = 2 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -esam_enabled = true -esam_rho = 0.0025 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2/report.json deleted file mode 100644 index 4a614a98f..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2/report.json +++ /dev/null @@ -1,178 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 18, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "esam_enabled": true, - "esam_rho": 0.0025, - "k": 16, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "esam_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.55613541574649, - "train_loss": 0.695133199817256, - "val_score": -11.682207347323509 - }, - { - "epoch": 1, - "test_score": -11.55939767819315, - "train_loss": 0.5532737437047457, - "val_score": -11.693448597924595 - }, - { - "epoch": 2, - "test_score": -11.555845583022986, - "train_loss": 0.5430088105954622, - "val_score": -11.695129189236418 - }, - { - "epoch": 3, - "test_score": -11.546128348237966, - "train_loss": 0.5180052236506814, - "val_score": -11.670440136243055 - }, - { - "epoch": 4, - "test_score": -11.540655833207525, - "train_loss": 0.5016618543549588, - "val_score": -11.668992019701589 - }, - { - "epoch": 5, - "test_score": -11.548262454700978, - "train_loss": 0.4916326858495411, - "val_score": -11.66722003895958 - }, - { - "epoch": 6, - "test_score": -11.580033818065433, - "train_loss": 0.47983503184820475, - "val_score": -11.701952805531674 - }, - { - "epoch": 7, - "test_score": -11.56019363082758, - "train_loss": 0.4752620725255263, - "val_score": -11.688177568511135 - }, - { - "epoch": 8, - "test_score": -11.54976600310857, - "train_loss": 0.46754956245422363, - "val_score": -11.678680192766727 - }, - { - "epoch": 9, - "test_score": -11.554276795016612, - "train_loss": 0.4644645609353718, - "val_score": -11.687640347377585 - }, - { - "epoch": 10, - "test_score": -11.528830173564812, - "train_loss": 0.45745536214426946, - "val_score": -11.665889241791978 - }, - { - "epoch": 11, - "test_score": -11.519614037300007, - "train_loss": 0.451346019380971, - "val_score": -11.658393396891343 - }, - { - "epoch": 12, - "test_score": -11.500646821780762, - "train_loss": 0.45580658002903585, - "val_score": -11.637033448665152 - }, - { - "epoch": 13, - "test_score": -11.525748576514388, - "train_loss": 0.45295710469547074, - "val_score": -11.663883276838083 - }, - { - "epoch": 14, - "test_score": -11.52637409616257, - "train_loss": 0.4448630213737488, - "val_score": -11.680014758166044 - }, - { - "epoch": 15, - "test_score": -11.498340652831377, - "train_loss": 0.443443437940196, - "val_score": -11.637611029696586 - }, - { - "epoch": 16, - "test_score": -11.527711071960198, - "train_loss": 0.43620154889006363, - "val_score": -11.678399280964193 - }, - { - "epoch": 17, - "test_score": -11.477502461081881, - "train_loss": 0.43538482251920196, - "val_score": -11.642487529267129 - }, - { - "epoch": 18, - "test_score": -11.461513970249932, - "train_loss": 0.4357229392779501, - "val_score": -11.629880475441134 - }, - { - "epoch": 19, - "test_score": -11.502000054295923, - "train_loss": 0.42684777473148544, - "val_score": -11.671822053720273 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.461513970249932, - "score": -11.461513970249932 - }, - "val": { - "rmse": 11.629880475441134, - "score": -11.629880475441134 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/esam_only-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 16.751631021499634, - "variant": "esam_only" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0.toml deleted file mode 100644 index a65899ec2..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "sberbank-housing" -variant = "mfb_only" -seed = 0 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0/report.json deleted file mode 100644 index 9f52a6eb1..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0/report.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 3, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "mfb_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.584012530789, - "train_loss": 0.7067123651504517, - "val_score": -11.712726316433415 - }, - { - "epoch": 1, - "test_score": -11.555901041327463, - "train_loss": 0.5641985504250777, - "val_score": -11.6994959013499 - }, - { - "epoch": 2, - "test_score": -11.536238916978505, - "train_loss": 0.5335533916950226, - "val_score": -11.668051137974945 - }, - { - "epoch": 3, - "test_score": -11.527389417691648, - "train_loss": 0.5168953067377994, - "val_score": -11.649721173400192 - }, - { - "epoch": 4, - "test_score": -11.563132768791982, - "train_loss": 0.508146091511375, - "val_score": -11.67863968962139 - }, - { - "epoch": 5, - "test_score": -11.572309632036578, - "train_loss": 0.5049841780411569, - "val_score": -11.674906917748249 - }, - { - "epoch": 6, - "test_score": -11.562115307156237, - "train_loss": 0.5046039474637884, - "val_score": -11.671406972711399 - }, - { - "epoch": 7, - "test_score": -11.570539988690363, - "train_loss": 0.5026436055961409, - "val_score": -11.692601686712054 - }, - { - "epoch": 8, - "test_score": -11.58557531674288, - "train_loss": 0.48302085462369415, - "val_score": -11.692135141620707 - }, - { - "epoch": 9, - "test_score": -11.574419136385487, - "train_loss": 0.4848339212568183, - "val_score": -11.683977055104942 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.527389417691648, - "score": -11.527389417691648 - }, - "val": { - "rmse": 11.649721173400192, - "score": -11.649721173400192 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 6.568762302398682, - "variant": "mfb_only" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1.toml deleted file mode 100644 index 709c7b9e6..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "sberbank-housing" -variant = "mfb_only" -seed = 1 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1/report.json deleted file mode 100644 index c300034d8..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1/report.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 1, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "mfb_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.582838820918091, - "train_loss": 0.7015442032563058, - "val_score": -11.693712184891757 - }, - { - "epoch": 1, - "test_score": -11.535810359671178, - "train_loss": 0.5580546150082036, - "val_score": -11.652520531679441 - }, - { - "epoch": 2, - "test_score": -11.555896419812255, - "train_loss": 0.5397704180918241, - "val_score": -11.676539866413899 - }, - { - "epoch": 3, - "test_score": -11.589493538271325, - "train_loss": 0.5239935846705186, - "val_score": -11.7085437325865 - }, - { - "epoch": 4, - "test_score": -11.605585108597094, - "train_loss": 0.5143824502041465, - "val_score": -11.710058625919041 - }, - { - "epoch": 5, - "test_score": -11.556786357508125, - "train_loss": 0.5057109406119898, - "val_score": -11.657854803233795 - }, - { - "epoch": 6, - "test_score": -11.586644054020574, - "train_loss": 0.4948484772130063, - "val_score": -11.70785235483841 - }, - { - "epoch": 7, - "test_score": -11.586848834690507, - "train_loss": 0.497083875693773, - "val_score": -11.705622204159992 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.535810359671178, - "score": -11.535810359671178 - }, - "val": { - "rmse": 11.652520531679441, - "score": -11.652520531679441 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 5.5788445472717285, - "variant": "mfb_only" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2.toml deleted file mode 100644 index db0d6a2a9..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2.toml +++ /dev/null @@ -1,29 +0,0 @@ -dataset = "sberbank-housing" -variant = "mfb_only" -seed = 2 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -mfb_enabled = true -mfb_keep = 0.8 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2/report.json deleted file mode 100644 index bcd4bfc6e..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2/report.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 4, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "mfb_enabled": true, - "mfb_keep": 0.8, - "n_bins": 48, - "n_blocks": 2 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "mfb_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.548783698174738, - "train_loss": 0.7081226235941837, - "val_score": -11.686633725624931 - }, - { - "epoch": 1, - "test_score": -11.547962514332209, - "train_loss": 0.5557862222194672, - "val_score": -11.684425643235729 - }, - { - "epoch": 2, - "test_score": -11.556265474993019, - "train_loss": 0.5497071068537863, - "val_score": -11.701433820980952 - }, - { - "epoch": 3, - "test_score": -11.557251764990456, - "train_loss": 0.5295571292701521, - "val_score": -11.683871924973317 - }, - { - "epoch": 4, - "test_score": -11.513798272472021, - "train_loss": 0.5162518667547327, - "val_score": -11.642743095889173 - }, - { - "epoch": 5, - "test_score": -11.543931722733838, - "train_loss": 0.5097392264165377, - "val_score": -11.66748356456919 - }, - { - "epoch": 6, - "test_score": -11.554824838602325, - "train_loss": 0.49350636099514206, - "val_score": -11.682919180835649 - }, - { - "epoch": 7, - "test_score": -11.545274595197622, - "train_loss": 0.49634232646540594, - "val_score": -11.684635892589156 - }, - { - "epoch": 8, - "test_score": -11.592842498385926, - "train_loss": 0.48988168176851776, - "val_score": -11.717275297835817 - }, - { - "epoch": 9, - "test_score": -11.546218213306766, - "train_loss": 0.48752505528299434, - "val_score": -11.674862480578934 - }, - { - "epoch": 10, - "test_score": -11.546426353978601, - "train_loss": 0.4796864672711021, - "val_score": -11.681482408067907 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.513798272472021, - "score": -11.513798272472021 - }, - "val": { - "rmse": 11.642743095889173, - "score": -11.642743095889173 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/mfb_only-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 8.56472897529602, - "variant": "mfb_only" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0.toml deleted file mode 100644 index 2856da2cf..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0.toml +++ /dev/null @@ -1,30 +0,0 @@ -dataset = "sberbank-housing" -variant = "rla_only" -seed = 0 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0/report.json deleted file mode 100644 index 7c088e464..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0/report.json +++ /dev/null @@ -1,113 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 2, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 0, - "variant": "rla_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.520820677453115, - "train_loss": 0.7019921698068318, - "val_score": -11.648102149766984 - }, - { - "epoch": 1, - "test_score": -11.536492208024947, - "train_loss": 0.5585583950343885, - "val_score": -11.659057606323854 - }, - { - "epoch": 2, - "test_score": -11.49868567875769, - "train_loss": 0.532463694873609, - "val_score": -11.630861178893646 - }, - { - "epoch": 3, - "test_score": -11.549928501452806, - "train_loss": 0.5084077339423331, - "val_score": -11.677087398713239 - }, - { - "epoch": 4, - "test_score": -11.576383925687757, - "train_loss": 0.4942305793887691, - "val_score": -11.696462530088892 - }, - { - "epoch": 5, - "test_score": -11.54979969197755, - "train_loss": 0.492362883530165, - "val_score": -11.66206145448411 - }, - { - "epoch": 6, - "test_score": -11.59433500145552, - "train_loss": 0.4821073114871979, - "val_score": -11.703687585568316 - }, - { - "epoch": 7, - "test_score": -11.524535832191289, - "train_loss": 0.48153042793273926, - "val_score": -11.644755323238266 - }, - { - "epoch": 8, - "test_score": -11.552281828889978, - "train_loss": 0.46902895601172195, - "val_score": -11.671850161034481 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.49868567875769, - "score": -11.49868567875769 - }, - "val": { - "rmse": 11.630861178893646, - "score": -11.630861178893646 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/0", - "seed": 0, - "task_type": "regression", - "time_seconds": 6.48681378364563, - "variant": "rla_only" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1.toml deleted file mode 100644 index ecff84b3e..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1.toml +++ /dev/null @@ -1,30 +0,0 @@ -dataset = "sberbank-housing" -variant = "rla_only" -seed = 1 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1/report.json deleted file mode 100644 index fa541d4ce..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1/report.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 1, - "variant": "rla_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.506723180155442, - "train_loss": 0.6959314220830014, - "val_score": -11.633652615105254 - }, - { - "epoch": 1, - "test_score": -11.540293551038493, - "train_loss": 0.5563824114046598, - "val_score": -11.683249613579397 - }, - { - "epoch": 2, - "test_score": -11.562496700801894, - "train_loss": 0.5268118506983707, - "val_score": -11.710368096147347 - }, - { - "epoch": 3, - "test_score": -11.520426645855572, - "train_loss": 0.5095094709019912, - "val_score": -11.652476009138773 - }, - { - "epoch": 4, - "test_score": -11.552124647107332, - "train_loss": 0.4992134774986066, - "val_score": -11.681942847672602 - }, - { - "epoch": 5, - "test_score": -11.545625487408978, - "train_loss": 0.49524260978949697, - "val_score": -11.674771645250711 - }, - { - "epoch": 6, - "test_score": -11.599754567673983, - "train_loss": 0.48614214439141124, - "val_score": -11.717217998818489 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.506723180155442, - "score": -11.506723180155442 - }, - "val": { - "rmse": 11.633652615105254, - "score": -11.633652615105254 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/1", - "seed": 1, - "task_type": "regression", - "time_seconds": 5.13034200668335, - "variant": "rla_only" -} diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2.toml deleted file mode 100644 index 8c2083ffb..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2.toml +++ /dev/null @@ -1,30 +0,0 @@ -dataset = "sberbank-housing" -variant = "rla_only" -seed = 2 -batch_size = 1024 -patience = 6 -n_epochs = 20 -gradient_clipping_norm = 1.0 -amp = true - -[data] -path = "data/sberbank-housing" -num_policy = "standard" -cache = false - -[optimizer] -type = "AdamW" -lr = 0.0003 -weight_decay = 1e-05 - -[model] -arch_type = "tabm" -k = 16 -n_blocks = 2 -d_block = 384 -dropout = 0.1 -d_embedding = 16 -n_bins = 48 -rla_enabled = true -rla_rank = 4 -rla_scale = 0.05 diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2/DONE deleted file mode 100644 index 19f86f493..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2/DONE +++ /dev/null @@ -1 +0,0 @@ -done diff --git a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2/report.json deleted file mode 100644 index 5b56c9ef9..000000000 --- a/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2/report.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "amp_dtype": "bfloat16", - "amp_enabled": true, - "best_epoch": 0, - "config": { - "amp": true, - "batch_size": 1024, - "data": { - "cache": false, - "num_policy": "standard", - "path": "data/sberbank-housing" - }, - "dataset": "sberbank-housing", - "gradient_clipping_norm": 1.0, - "model": { - "arch_type": "tabm", - "d_block": 384, - "d_embedding": 16, - "dropout": 0.1, - "k": 16, - "n_bins": 48, - "n_blocks": 2, - "rla_enabled": true, - "rla_rank": 4, - "rla_scale": 0.05 - }, - "n_epochs": 20, - "optimizer": { - "lr": 0.0003, - "type": "AdamW", - "weight_decay": 1e-05 - }, - "patience": 6, - "seed": 2, - "variant": "rla_only" - }, - "config_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2.toml", - "dataset": "sberbank-housing", - "gpu_name": "NVIDIA GeForce RTX 4090", - "history": [ - { - "epoch": 0, - "test_score": -11.506943969685626, - "train_loss": 0.693781990753977, - "val_score": -11.638517007119077 - }, - { - "epoch": 1, - "test_score": -11.548524730973826, - "train_loss": 0.5583565109654477, - "val_score": -11.691600059995665 - }, - { - "epoch": 2, - "test_score": -11.5435219569212, - "train_loss": 0.5326070801207894, - "val_score": -11.68417294723668 - }, - { - "epoch": 3, - "test_score": -11.558364706035814, - "train_loss": 0.5135152920296318, - "val_score": -11.696133775230425 - }, - { - "epoch": 4, - "test_score": -11.55967818185129, - "train_loss": 0.49890078212085526, - "val_score": -11.692484236554279 - }, - { - "epoch": 5, - "test_score": -11.546965519985177, - "train_loss": 0.4914465530922538, - "val_score": -11.677945234810213 - }, - { - "epoch": 6, - "test_score": -11.532360819421838, - "train_loss": 0.4807174880253641, - "val_score": -11.65013244291669 - } - ], - "inference_mode": "mean", - "metric_direction": "lower", - "metrics": { - "test": { - "rmse": 11.506943969685626, - "score": -11.506943969685626 - }, - "val": { - "rmse": 11.638517007119077, - "score": -11.638517007119077 - } - }, - "result_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sberbank-housing/rla_only-evaluation/2", - "seed": 2, - "task_type": "regression", - "time_seconds": 5.9000396728515625, - "variant": "rla_only" -} diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6bf7e0a6e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-best-head-evaluation/0/report.json @@ -0,0 +1,455 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4515053148590503, + "mae": 0.35153692960739136, + "r2": 0.5317407250404358, + "score": -0.4515053148590503 + }, + "val": { + "rmse": 0.46391938568359065, + "mae": 0.36074763536453247, + "r2": 0.5064277052879333, + "score": -0.46391938568359065 + }, + "test": { + "rmse": 0.48142889277254225, + "mae": 0.3758453130722046, + "r2": 0.4832448363304138, + "score": -0.48142889277254225 + } + }, + "time": "0:38:28.268413", + "diversity": { + "train": { + "mean_centered_corr": -0.009637685492634773, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04498574882745743 + }, + "val": { + "mean_centered_corr": -0.009298564866185188, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046711746603250504 + }, + "test": { + "mean_centered_corr": -0.00908930879086256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04780808091163635 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.028059" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..7de7ee9ce --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0.toml @@ -0,0 +1,74 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..eca7ffcc8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/0/report.json @@ -0,0 +1,451 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44819067366049176, + "mae": 0.3488732874393463, + "r2": 0.5385907888412476, + "score": -0.44819067366049176 + }, + "val": { + "rmse": 0.463355012524859, + "mae": 0.3601790964603424, + "r2": 0.5076278448104858, + "score": -0.463355012524859 + }, + "test": { + "rmse": 0.4809202538385541, + "mae": 0.3753686547279358, + "r2": 0.48433613777160645, + "score": -0.4809202538385541 + } + }, + "time": "0:38:28.268413", + "diversity": { + "train": { + "mean_centered_corr": -0.009637685492634773, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04498574882745743 + }, + "val": { + "mean_centered_corr": -0.009298564866185188, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046711746603250504 + }, + "test": { + "mean_centered_corr": -0.00908930879086256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04780808091163635 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f61c80709 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,464 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/cooking-time/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44759809638897813, + "mae": 0.3485293388366699, + "r2": 0.539810061454773, + "score": -0.44759809638897813 + }, + "val": { + "rmse": 0.4629352081332738, + "mae": 0.35992681980133057, + "r2": 0.5085196495056152, + "score": -0.4629352081332738 + }, + "test": { + "rmse": 0.4802514336856013, + "mae": 0.3748669922351837, + "r2": 0.48576945066452026, + "score": -0.4802514336856013 + } + }, + "time": "0:38:28.268413", + "diversity": { + "train": { + "mean_centered_corr": -0.009637685492634773, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04498574882745743 + }, + "val": { + "mean_centered_corr": -0.009298564866185188, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046711746603250504 + }, + "test": { + "mean_centered_corr": -0.00908930879086256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04780808091163635 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 25, + 24, + 19, + 1, + 21, + 27, + 15, + 2, + 12, + 31 + ], + "head_selection_time": "0:00:00.356165" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..dca12e41a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,147 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44650602179756516, + "mae": 0.34757497906684875, + "r2": 0.5420529246330261, + "score": -0.44650602179756516 + }, + "val": { + "rmse": 0.4652274815891859, + "mae": 0.3617282807826996, + "r2": 0.503640353679657, + "score": -0.4652274815891859 + }, + "test": { + "rmse": 0.4820660657381116, + "mae": 0.3761262893676758, + "r2": 0.48187607526779175, + "score": -0.4820660657381116 + } + }, + "time": "0:09:50.236903", + "diversity": { + "train": { + "mean_centered_corr": -0.020426616072654724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051189813762903214 + }, + "val": { + "mean_centered_corr": -0.02039431221783161, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05273108929395676 + }, + "test": { + "mean_centered_corr": -0.020482471212744713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053407639265060425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27 + ], + "head_selection_time": "0:00:00.028691" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..78c613dfb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..2c476e8cd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0/report.json @@ -0,0 +1,143 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44389690465868414, + "mae": 0.34541288018226624, + "r2": 0.5473892688751221, + "score": -0.44389690465868414 + }, + "val": { + "rmse": 0.4627596029926704, + "mae": 0.35959550738334656, + "r2": 0.5088924169540405, + "score": -0.4627596029926704 + }, + "test": { + "rmse": 0.4802386655626689, + "mae": 0.37469902634620667, + "r2": 0.4857967495918274, + "score": -0.4802386655626689 + } + }, + "time": "0:09:50.236903", + "diversity": { + "train": { + "mean_centered_corr": -0.020426616072654724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051189813762903214 + }, + "val": { + "mean_centered_corr": -0.02039431221783161, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05273108929395676 + }, + "test": { + "mean_centered_corr": -0.020482471212744713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053407639265060425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..81f6192e3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44394242184625543, + "mae": 0.3455282151699066, + "r2": 0.547296404838562, + "score": -0.44394242184625543 + }, + "val": { + "rmse": 0.4625169879783321, + "mae": 0.3595449924468994, + "r2": 0.5094072818756104, + "score": -0.4625169879783321 + }, + "test": { + "rmse": 0.47990786370986477, + "mae": 0.3745970129966736, + "r2": 0.4865049123764038, + "score": -0.47990786370986477 + } + }, + "time": "0:09:50.236903", + "diversity": { + "train": { + "mean_centered_corr": -0.020426616072654724, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051189813762903214 + }, + "val": { + "mean_centered_corr": -0.02039431221783161, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05273108929395676 + }, + "test": { + "mean_centered_corr": -0.020482471212744713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053407639265060425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27, + 17, + 5, + 3, + 14, + 4, + 22, + 6, + 24, + 10, + 16 + ], + "head_selection_time": "0:00:00.401983" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ac492b73b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-best-head-evaluation/0/report.json @@ -0,0 +1,400 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.45293358344227996, + "mae": 0.35263803601264954, + "r2": 0.5287735462188721, + "score": -0.45293358344227996 + }, + "val": { + "rmse": 0.46397138529545745, + "mae": 0.3607122600078583, + "r2": 0.5063170194625854, + "score": -0.46397138529545745 + }, + "test": { + "rmse": 0.4813020969034895, + "mae": 0.3759235143661499, + "r2": 0.48351699113845825, + "score": -0.4813020969034895 + } + }, + "time": "0:10:10.940252", + "diversity": { + "train": { + "mean_centered_corr": -0.004884317051619291, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042778123170137405 + }, + "val": { + "mean_centered_corr": -0.005044749937951565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04371502622961998 + }, + "test": { + "mean_centered_corr": -0.005197949707508087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04438793286681175 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 2 + ], + "head_selection_time": "0:00:00.027493" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..a22753f01 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..6635035ec --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,396 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4484508087097321, + "mae": 0.34904322028160095, + "r2": 0.5380549430847168, + "score": -0.4484508087097321 + }, + "val": { + "rmse": 0.4632809116304394, + "mae": 0.36012470722198486, + "r2": 0.5077853202819824, + "score": -0.4632809116304394 + }, + "test": { + "rmse": 0.48091704691326154, + "mae": 0.3753390908241272, + "r2": 0.4843430519104004, + "score": -0.48091704691326154 + } + }, + "time": "0:10:10.940252", + "diversity": { + "train": { + "mean_centered_corr": -0.004884317051619291, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042778123170137405 + }, + "val": { + "mean_centered_corr": -0.005044749937951565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04371502622961998 + }, + "test": { + "mean_centered_corr": -0.005197949707508087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04438793286681175 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8e51c0a26 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/cf_fisd_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,403 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44943641696524117, + "mae": 0.34985706210136414, + "r2": 0.5360222458839417, + "score": -0.44943641696524117 + }, + "val": { + "rmse": 0.4629530240908241, + "mae": 0.35998642444610596, + "r2": 0.5084818601608276, + "score": -0.4629530240908241 + }, + "test": { + "rmse": 0.480628211999089, + "mae": 0.37514716386795044, + "r2": 0.48496222496032715, + "score": -0.480628211999089 + } + }, + "time": "0:10:10.940252", + "diversity": { + "train": { + "mean_centered_corr": -0.004884317051619291, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042778123170137405 + }, + "val": { + "mean_centered_corr": -0.005044749937951565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04371502622961998 + }, + "test": { + "mean_centered_corr": -0.005197949707508087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04438793286681175 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 2, + 14, + 24, + 31 + ], + "head_selection_time": "0:00:00.139986" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7a1f0e5ca --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-best-head-evaluation/0/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4481166090239535, + "mae": 0.3489066958427429, + "r2": 0.538743257522583, + "score": -0.4481166090239535 + }, + "val": { + "rmse": 0.4651283707110853, + "mae": 0.3615894913673401, + "r2": 0.5038518309593201, + "score": -0.4651283707110853 + }, + "test": { + "rmse": 0.4823068025931435, + "mae": 0.37674346566200256, + "r2": 0.48135846853256226, + "score": -0.4823068025931435 + } + }, + "time": "0:15:54.265201", + "diversity": { + "train": { + "mean_centered_corr": -0.020383337512612343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05032477155327797 + }, + "val": { + "mean_centered_corr": -0.02031499706208706, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051892586052417755 + }, + "test": { + "mean_centered_corr": -0.02030996046960354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05258215218782425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 3 + ], + "head_selection_time": "0:00:00.027704" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/0.toml new file mode 100644 index 000000000..ae52f3ec3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/0/report.json new file mode 100644 index 000000000..94fa2204e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/0/report.json @@ -0,0 +1,153 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44408822333443154, + "mae": 0.3455769121646881, + "r2": 0.5469989776611328, + "score": -0.44408822333443154 + }, + "val": { + "rmse": 0.4628077726618535, + "mae": 0.3596377372741699, + "r2": 0.5087901949882507, + "score": -0.4628077726618535 + }, + "test": { + "rmse": 0.48022789849772834, + "mae": 0.37472617626190186, + "r2": 0.48581981658935547, + "score": -0.48022789849772834 + } + }, + "time": "0:15:54.265201", + "diversity": { + "train": { + "mean_centered_corr": -0.020383337512612343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05032477155327797 + }, + "val": { + "mean_centered_corr": -0.02031499706208706, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051892586052417755 + }, + "test": { + "mean_centered_corr": -0.02030996046960354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05258215218782425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..398e51916 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,167 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/cooking-time/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4439705320848652, + "mae": 0.34557339549064636, + "r2": 0.5472390651702881, + "score": -0.4439705320848652 + }, + "val": { + "rmse": 0.4625398779765968, + "mae": 0.35953423380851746, + "r2": 0.5093587040901184, + "score": -0.4625398779765968 + }, + "test": { + "rmse": 0.4797024392153251, + "mae": 0.3744461238384247, + "r2": 0.48694443702697754, + "score": -0.4797024392153251 + } + }, + "time": "0:15:54.265201", + "diversity": { + "train": { + "mean_centered_corr": -0.020383337512612343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05032477155327797 + }, + "val": { + "mean_centered_corr": -0.02031499706208706, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051892586052417755 + }, + "test": { + "mean_centered_corr": -0.02030996046960354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05258215218782425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 3, + 17, + 5, + 27, + 22, + 14, + 10, + 4, + 24, + 25, + 20 + ], + "head_selection_time": "0:00:00.400987" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3f8cf5f28 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-best-head-evaluation/0/report.json @@ -0,0 +1,183 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44680861066629424, + "mae": 0.3479525148868561, + "r2": 0.5414320230484009, + "score": -0.44680861066629424 + }, + "val": { + "rmse": 0.465403548325863, + "mae": 0.3621026575565338, + "r2": 0.5032646059989929, + "score": -0.465403548325863 + }, + "test": { + "rmse": 0.48311284423774803, + "mae": 0.377098023891449, + "r2": 0.4796234369277954, + "score": -0.48311284423774803 + } + }, + "time": "0:12:03.727238", + "diversity": { + "train": { + "mean_centered_corr": -0.020108899101614952, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05420300364494324 + }, + "val": { + "mean_centered_corr": -0.01979835145175457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05661233514547348 + }, + "test": { + "mean_centered_corr": -0.01982581429183483, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05757860466837883 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 17 + ], + "head_selection_time": "0:00:00.027716" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-evaluation/0.toml new file mode 100644 index 000000000..349df099f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..27ffe0cdb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4438062088250124, + "mae": 0.34538015723228455, + "r2": 0.5475741624832153, + "score": -0.4438062088250124 + }, + "val": { + "rmse": 0.46290981075910853, + "mae": 0.3597574234008789, + "r2": 0.508573591709137, + "score": -0.46290981075910853 + }, + "test": { + "rmse": 0.480415480847424, + "mae": 0.37490591406822205, + "r2": 0.48541808128356934, + "score": -0.480415480847424 + } + }, + "time": "0:12:03.727238", + "diversity": { + "train": { + "mean_centered_corr": -0.020108899101614952, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05420300364494324 + }, + "val": { + "mean_centered_corr": -0.01979835145175457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05661233514547348 + }, + "test": { + "mean_centered_corr": -0.01982581429183483, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05757860466837883 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..be582a713 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/mfb_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,193 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4439127657129147, + "mae": 0.3454906642436981, + "r2": 0.5473568439483643, + "score": -0.4439127657129147 + }, + "val": { + "rmse": 0.4625593198933149, + "mae": 0.35958266258239746, + "r2": 0.5093174576759338, + "score": -0.4625593198933149 + }, + "test": { + "rmse": 0.479899744054174, + "mae": 0.37457266449928284, + "r2": 0.4865223169326782, + "score": -0.479899744054174 + } + }, + "time": "0:12:03.727238", + "diversity": { + "train": { + "mean_centered_corr": -0.020108899101614952, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05420300364494324 + }, + "val": { + "mean_centered_corr": -0.01979835145175457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05661233514547348 + }, + "test": { + "mean_centered_corr": -0.01982581429183483, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05757860466837883 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 17, + 14, + 11, + 24, + 25, + 5, + 4, + 27, + 20, + 1, + 9 + ], + "head_selection_time": "0:00:00.396117" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..30254cc4d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-best-head-evaluation/0/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44760284038640463, + "mae": 0.3484247922897339, + "r2": 0.5398002862930298, + "score": -0.44760284038640463 + }, + "val": { + "rmse": 0.4651183431272534, + "mae": 0.3614919185638428, + "r2": 0.5038732290267944, + "score": -0.4651183431272534 + }, + "test": { + "rmse": 0.48213882482447107, + "mae": 0.37617027759552, + "r2": 0.48171961307525635, + "score": -0.48213882482447107 + } + }, + "time": "0:23:04.537939", + "diversity": { + "train": { + "mean_centered_corr": -0.020404240116477013, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513404980301857 + }, + "val": { + "mean_centered_corr": -0.02041091024875641, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052927520126104355 + }, + "test": { + "mean_centered_corr": -0.020323792472481728, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05353504791855812 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7 + ], + "head_selection_time": "0:00:00.027790" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-evaluation/0.toml new file mode 100644 index 000000000..17b62a97a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-evaluation/0/report.json new file mode 100644 index 000000000..ae0abfdfd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-evaluation/0/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4435648840971451, + "mae": 0.34517785906791687, + "r2": 0.5480660200119019, + "score": -0.4435648840971451 + }, + "val": { + "rmse": 0.4627297681263241, + "mae": 0.35958367586135864, + "r2": 0.5089557766914368, + "score": -0.4627297681263241 + }, + "test": { + "rmse": 0.48025858554781287, + "mae": 0.3747011125087738, + "r2": 0.485754132270813, + "score": -0.48025858554781287 + } + }, + "time": "0:23:04.537939", + "diversity": { + "train": { + "mean_centered_corr": -0.020404240116477013, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513404980301857 + }, + "val": { + "mean_centered_corr": -0.02041091024875641, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052927520126104355 + }, + "test": { + "mean_centered_corr": -0.020323792472481728, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05353504791855812 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..551d9ab7f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/cooking-time/rla_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,177 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44353548828929634, + "mae": 0.3452431261539459, + "r2": 0.5481259822845459, + "score": -0.44353548828929634 + }, + "val": { + "rmse": 0.4624375326912171, + "mae": 0.3594800531864166, + "r2": 0.5095758438110352, + "score": -0.4624375326912171 + }, + "test": { + "rmse": 0.48004674599435365, + "mae": 0.3747219443321228, + "r2": 0.48620766401290894, + "score": -0.48004674599435365 + } + }, + "time": "0:23:04.537939", + "diversity": { + "train": { + "mean_centered_corr": -0.020404240116477013, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513404980301857 + }, + "val": { + "mean_centered_corr": -0.02041091024875641, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052927520126104355 + }, + "test": { + "mean_centered_corr": -0.020323792472481728, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05353504791855812 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7, + 24, + 17, + 14, + 5, + 3, + 16, + 4, + 10, + 6, + 22, + 17, + 27, + 24, + 21, + 5, + 14, + 31, + 17, + 25, + 24, + 28 + ], + "head_selection_time": "0:00:01.021222" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..487266694 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-best-head-evaluation/0/report.json @@ -0,0 +1,506 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5163498609634084, + "mae": 0.40700992941856384, + "r2": 0.500271201133728, + "score": -0.5163498609634084 + }, + "val": { + "rmse": 0.557107642501001, + "mae": 0.44366100430488586, + "r2": 0.4147301912307739, + "score": -0.557107642501001 + }, + "test": { + "rmse": 0.5520572896128711, + "mae": 0.4460996389389038, + "r2": 0.43209290504455566, + "score": -0.5520572896128711 + } + }, + "time": "0:30:03.338423", + "diversity": { + "train": { + "mean_centered_corr": -0.019723476842045784, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059378866106271744 + }, + "val": { + "mean_centered_corr": -0.019546858966350555, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06028679013252258 + }, + "test": { + "mean_centered_corr": -0.0195509921759367, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05971246585249901 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.023549" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..48f767b9b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0.toml @@ -0,0 +1,74 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..33856da09 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/0/report.json @@ -0,0 +1,502 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5114855052725168, + "mae": 0.4018738567829132, + "r2": 0.5096423625946045, + "score": -0.5114855052725168 + }, + "val": { + "rmse": 0.5560789252392828, + "mae": 0.44038042426109314, + "r2": 0.41688960790634155, + "score": -0.5560789252392828 + }, + "test": { + "rmse": 0.5493995976501785, + "mae": 0.44166451692581177, + "r2": 0.4375477433204651, + "score": -0.5493995976501785 + } + }, + "time": "0:30:03.338423", + "diversity": { + "train": { + "mean_centered_corr": -0.019723476842045784, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059378866106271744 + }, + "val": { + "mean_centered_corr": -0.019546858966350555, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06028679013252258 + }, + "test": { + "mean_centered_corr": -0.0195509921759367, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05971246585249901 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2023ea3bd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,510 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5123158821979983, + "mae": 0.40332356095314026, + "r2": 0.5080488920211792, + "score": -0.5123158821979983 + }, + "val": { + "rmse": 0.5558210261849236, + "mae": 0.4417085349559784, + "r2": 0.41743040084838867, + "score": -0.5558210261849236 + }, + "test": { + "rmse": 0.551614874495521, + "mae": 0.4454870820045471, + "r2": 0.433002769947052, + "score": -0.551614874495521 + } + }, + "time": "0:30:03.338423", + "diversity": { + "train": { + "mean_centered_corr": -0.019723476842045784, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059378866106271744 + }, + "val": { + "mean_centered_corr": -0.019546858966350555, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06028679013252258 + }, + "test": { + "mean_centered_corr": -0.0195509921759367, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05971246585249901 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 24, + 30, + 17, + 8, + 12 + ], + "head_selection_time": "0:00:00.146995" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..55ac8b264 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,168 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:16.600998", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023548" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..d07370fc8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..5bbb360c8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0/report.json @@ -0,0 +1,164 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:16.600998", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..fc04b59cb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,181 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:16.600998", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.432507" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..873cd222c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-best-head-evaluation/0/report.json @@ -0,0 +1,449 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3003, + "metrics": { + "train": { + "rmse": 0.5091186451729569, + "mae": 0.40107491612434387, + "r2": 0.5141700506210327, + "score": -0.5091186451729569 + }, + "val": { + "rmse": 0.5580564462352832, + "mae": 0.4435662627220154, + "r2": 0.4127349853515625, + "score": -0.5580564462352832 + }, + "test": { + "rmse": 0.5528824139554394, + "mae": 0.4462261497974396, + "r2": 0.4303940534591675, + "score": -0.5528824139554394 + } + }, + "time": "0:09:03.165444", + "diversity": { + "train": { + "mean_centered_corr": -0.017171135172247887, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07136442512273788 + }, + "val": { + "mean_centered_corr": -0.01788639836013317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07444944232702255 + }, + "test": { + "mean_centered_corr": -0.017493220046162605, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07251272350549698 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 27 + ], + "head_selection_time": "0:00:00.023957" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..26f1234c9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..fdaec3b4f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,445 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3003, + "metrics": { + "train": { + "rmse": 0.5000776588212498, + "mae": 0.39380842447280884, + "r2": 0.5312716960906982, + "score": -0.5000776588212498 + }, + "val": { + "rmse": 0.5562691235201772, + "mae": 0.44062381982803345, + "r2": 0.41649067401885986, + "score": -0.5562691235201772 + }, + "test": { + "rmse": 0.5515996385557942, + "mae": 0.4439227879047394, + "r2": 0.4330340623855591, + "score": -0.5515996385557942 + } + }, + "time": "0:09:03.165444", + "diversity": { + "train": { + "mean_centered_corr": -0.017171135172247887, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07136442512273788 + }, + "val": { + "mean_centered_corr": -0.01788639836013317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07444944232702255 + }, + "test": { + "mean_centered_corr": -0.017493220046162605, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07251272350549698 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..cd4f2f046 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/cf_fisd_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,469 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3003, + "metrics": { + "train": { + "rmse": 0.5022562202858329, + "mae": 0.39594367146492004, + "r2": 0.5271788239479065, + "score": -0.5022562202858329 + }, + "val": { + "rmse": 0.5559326756199947, + "mae": 0.4413962960243225, + "r2": 0.41719627380371094, + "score": -0.5559326756199947 + }, + "test": { + "rmse": 0.5522105562224877, + "mae": 0.4452051818370819, + "r2": 0.4317775368690491, + "score": -0.5522105562224877 + } + }, + "time": "0:09:03.165444", + "diversity": { + "train": { + "mean_centered_corr": -0.017171135172247887, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07136442512273788 + }, + "val": { + "mean_centered_corr": -0.01788639836013317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07444944232702255 + }, + "test": { + "mean_centered_corr": -0.017493220046162605, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07251272350549698 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 27, + 10, + 7, + 26, + 25, + 17, + 1, + 11, + 28, + 30, + 20, + 9, + 26, + 23, + 19, + 3, + 10, + 12, + 29, + 26, + 24 + ], + "head_selection_time": "0:00:00.737306" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6e8396567 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-best-head-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5123514530013489, + "mae": 0.4078504741191864, + "r2": 0.5079806447029114, + "score": -0.5123514530013489 + }, + "val": { + "rmse": 0.5582863288518123, + "mae": 0.4448302686214447, + "r2": 0.41225099563598633, + "score": -0.5582863288518123 + }, + "test": { + "rmse": 0.5560390501031047, + "mae": 0.4517032206058502, + "r2": 0.4238712191581726, + "score": -0.5560390501031047 + } + }, + "time": "0:12:51.556071", + "diversity": { + "train": { + "mean_centered_corr": -0.02027146704494953, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06810352951288223 + }, + "val": { + "mean_centered_corr": -0.02014581859111786, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07032498717308044 + }, + "test": { + "mean_centered_corr": -0.02043480984866619, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0694202110171318 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023277" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/0.toml new file mode 100644 index 000000000..4b93a10d8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/0/report.json new file mode 100644 index 000000000..239380f66 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.504551673775848, + "mae": 0.39544814825057983, + "r2": 0.5228471159934998, + "score": -0.504551673775848 + }, + "val": { + "rmse": 0.5556360918188752, + "mae": 0.439733624458313, + "r2": 0.41781800985336304, + "score": -0.5556360918188752 + }, + "test": { + "rmse": 0.5494649592968895, + "mae": 0.4416595995426178, + "r2": 0.43741387128829956, + "score": -0.5494649592968895 + } + }, + "time": "0:12:51.556071", + "diversity": { + "train": { + "mean_centered_corr": -0.02027146704494953, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06810352951288223 + }, + "val": { + "mean_centered_corr": -0.02014581859111786, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07032498717308044 + }, + "test": { + "mean_centered_corr": -0.02043480984866619, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0694202110171318 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..20d61ffa8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,190 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5047633843312425, + "mae": 0.3963008522987366, + "r2": 0.5224465727806091, + "score": -0.5047633843312425 + }, + "val": { + "rmse": 0.5551513290724005, + "mae": 0.4403177499771118, + "r2": 0.4188333749771118, + "score": -0.5551513290724005 + }, + "test": { + "rmse": 0.5507488105906536, + "mae": 0.44414353370666504, + "r2": 0.43478184938430786, + "score": -0.5507488105906536 + } + }, + "time": "0:12:51.556071", + "diversity": { + "train": { + "mean_centered_corr": -0.02027146704494953, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06810352951288223 + }, + "val": { + "mean_centered_corr": -0.02014581859111786, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07032498717308044 + }, + "test": { + "mean_centered_corr": -0.02043480984866619, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0694202110171318 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 21, + 10, + 6, + 29, + 12, + 17, + 15, + 8, + 22, + 16, + 21, + 4 + ], + "head_selection_time": "0:00:00.394226" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..296044dd1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-best-head-evaluation/0/report.json @@ -0,0 +1,204 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5157195206907822, + "mae": 0.4108550548553467, + "r2": 0.501490592956543, + "score": -0.5157195206907822 + }, + "val": { + "rmse": 0.5590040394051904, + "mae": 0.4459855556488037, + "r2": 0.41073888540267944, + "score": -0.5590040394051904 + }, + "test": { + "rmse": 0.5557457406296488, + "mae": 0.45139390230178833, + "r2": 0.42447882890701294, + "score": -0.5557457406296488 + } + }, + "time": "0:08:31.551433", + "diversity": { + "train": { + "mean_centered_corr": -0.02001963183283806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07454843819141388 + }, + "val": { + "mean_centered_corr": -0.020045898854732513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650969177484512 + }, + "test": { + "mean_centered_corr": -0.02007085457444191, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0762816071510315 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023499" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0.toml new file mode 100644 index 000000000..8328925cb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..9fedd8bb3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-evaluation/0/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.505005369677181, + "mae": 0.3960992991924286, + "r2": 0.521988570690155, + "score": -0.505005369677181 + }, + "val": { + "rmse": 0.5560026829864069, + "mae": 0.4405945837497711, + "r2": 0.4170495271682739, + "score": -0.5560026829864069 + }, + "test": { + "rmse": 0.5501201530906976, + "mae": 0.4423939883708954, + "r2": 0.43607139587402344, + "score": -0.5501201530906976 + } + }, + "time": "0:08:31.551433", + "diversity": { + "train": { + "mean_centered_corr": -0.02001963183283806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07454843819141388 + }, + "val": { + "mean_centered_corr": -0.020045898854732513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650969177484512 + }, + "test": { + "mean_centered_corr": -0.02007085457444191, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0762816071510315 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2a282e7b9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/mfb_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,217 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.505321820939252, + "mae": 0.3968241810798645, + "r2": 0.5213893055915833, + "score": -0.505321820939252 + }, + "val": { + "rmse": 0.5555016895512279, + "mae": 0.4406256079673767, + "r2": 0.418099582195282, + "score": -0.5555016895512279 + }, + "test": { + "rmse": 0.5503428181880856, + "mae": 0.44352859258651733, + "r2": 0.43561482429504395, + "score": -0.5503428181880856 + } + }, + "time": "0:08:31.551433", + "diversity": { + "train": { + "mean_centered_corr": -0.02001963183283806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07454843819141388 + }, + "val": { + "mean_centered_corr": -0.020045898854732513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650969177484512 + }, + "test": { + "mean_centered_corr": -0.02007085457444191, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0762816071510315 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 25, + 26, + 15, + 23, + 17, + 6, + 4, + 24, + 21, + 12, + 6, + 10 + ], + "head_selection_time": "0:00:00.433229" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..69cbaf349 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5153977731507096, + "mae": 0.4045717418193817, + "r2": 0.5021123886108398, + "score": -0.5153977731507096 + }, + "val": { + "rmse": 0.5573103773832353, + "mae": 0.44244086742401123, + "r2": 0.41430413722991943, + "score": -0.5573103773832353 + }, + "test": { + "rmse": 0.5532468973961396, + "mae": 0.4463353753089905, + "r2": 0.4296427369117737, + "score": -0.5532468973961396 + } + }, + "time": "0:14:42.930744", + "diversity": { + "train": { + "mean_centered_corr": -0.02064315602183342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056059885770082474 + }, + "val": { + "mean_centered_corr": -0.020947575569152832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05900698900222778 + }, + "test": { + "mean_centered_corr": -0.020849451422691345, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05796251446008682 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.023323" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-evaluation/0.toml new file mode 100644 index 000000000..9dd7c431e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-evaluation/0/report.json new file mode 100644 index 000000000..5f0d2f187 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5133709914722272, + "mae": 0.4036697745323181, + "r2": 0.5060205459594727, + "score": -0.5133709914722272 + }, + "val": { + "rmse": 0.5556926485294449, + "mae": 0.4411713182926178, + "r2": 0.4176994562149048, + "score": -0.5556926485294449 + }, + "test": { + "rmse": 0.5502675684403805, + "mae": 0.4438808858394623, + "r2": 0.43576914072036743, + "score": -0.5502675684403805 + } + }, + "time": "0:14:42.930744", + "diversity": { + "train": { + "mean_centered_corr": -0.02064315602183342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056059885770082474 + }, + "val": { + "mean_centered_corr": -0.020947575569152832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05900698900222778 + }, + "test": { + "mean_centered_corr": -0.020849451422691345, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05796251446008682 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..72b0aed82 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/rla_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,184 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5137274534378229, + "mae": 0.40429580211639404, + "r2": 0.5053342580795288, + "score": -0.5137274534378229 + }, + "val": { + "rmse": 0.5553051102575359, + "mae": 0.4410802721977234, + "r2": 0.4185113310813904, + "score": -0.5553051102575359 + }, + "test": { + "rmse": 0.5508335169889497, + "mae": 0.4448656141757965, + "r2": 0.43460792303085327, + "score": -0.5508335169889497 + } + }, + "time": "0:14:42.930744", + "diversity": { + "train": { + "mean_centered_corr": -0.02064315602183342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056059885770082474 + }, + "val": { + "mean_centered_corr": -0.020947575569152832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05900698900222778 + }, + "test": { + "mean_centered_corr": -0.020849451422691345, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05796251446008682 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 31, + 16, + 19, + 15, + 11, + 21 + ], + "head_selection_time": "0:00:00.170577" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a15083dfe --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-best-head-evaluation/0/report.json @@ -0,0 +1,445 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.788873839009288, + "recall": 0.9686263795008256, + "f1-score": 0.8695577986445486, + "support": 84179.0 + }, + "1": { + "precision": 0.5584350443069721, + "recall": 0.13273984579922105, + "f1-score": 0.21449442892463796, + "support": 25162.0 + }, + "accuracy": 0.7762687372531805, + "macro avg": { + "precision": 0.67365444165813, + "recall": 0.5506831126500233, + "f1-score": 0.5420261137845933, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7358443171254596, + "recall": 0.7762687372531805, + "f1-score": 0.7188119255604138, + "support": 109341.0 + }, + "cross-entropy": 0.4989015904618268, + "roc-auc": 0.6823993201798577, + "score": 0.6823993201798577 + }, + "val": { + "0": { + "precision": 0.6940704108113401, + "recall": 0.9134553421832711, + "f1-score": 0.7887927436632258, + "support": 15518.0 + }, + "1": { + "precision": 0.6500781657113079, + "recall": 0.28537115406611, + "f1-score": 0.39662983864557666, + "support": 8743.0 + }, + "accuracy": 0.6871110011953341, + "macro avg": { + "precision": 0.672074288261324, + "recall": 0.5994132481246905, + "f1-score": 0.5927112911544012, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.678216810427614, + "recall": 0.6871110011953341, + "f1-score": 0.6474679722783157, + "support": 24261.0 + }, + "cross-entropy": 0.6244923069797662, + "roc-auc": 0.6479868261151002, + "score": 0.6479868261151002 + }, + "test": { + "0": { + "precision": 0.6674781317049018, + "recall": 0.9559744711027065, + "f1-score": 0.7860926186889547, + "support": 16922.0 + }, + "1": { + "precision": 0.6642631816133393, + "recall": 0.15462079093674605, + "f1-score": 0.25085091899251194, + "support": 9533.0 + }, + "accuracy": 0.6672084672084672, + "macro avg": { + "precision": 0.6658706566591206, + "recall": 0.5552976310197263, + "f1-score": 0.5184717688407333, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6663196316397774, + "recall": 0.6672084672084672, + "f1-score": 0.593219470958613, + "support": 26455.0 + }, + "cross-entropy": 0.65875352245754, + "roc-auc": 0.6047175337399694, + "score": 0.6047175337399694 + } + }, + "time": "0:03:40.866933", + "diversity": { + "train": { + "mean_centered_corr": -0.010312517173588276, + "mean_pairwise_disagreement": 0.02666120948639223, + "member_std": 0.029578423127532005 + }, + "val": { + "mean_centered_corr": -0.012083624489605427, + "mean_pairwise_disagreement": 0.09328301030061521, + "member_std": 0.06631172448396683 + }, + "test": { + "mean_centered_corr": -0.010351695120334625, + "mean_pairwise_disagreement": 0.058351141012431336, + "member_std": 0.06402193009853363 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 22 + ], + "head_selection_time": "0:00:04.786499" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..b8da71126 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0.toml @@ -0,0 +1,74 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..bf4559d41 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/0/report.json @@ -0,0 +1,441 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7843377382851067, + "recall": 0.9814680621057509, + "f1-score": 0.8718993219534074, + "support": 84179.0 + }, + "1": { + "precision": 0.6104868913857678, + "recall": 0.09717033622128607, + "f1-score": 0.16765522679740802, + "support": 25162.0 + }, + "accuracy": 0.7779698374809083, + "macro avg": { + "precision": 0.6974123148354372, + "recall": 0.5393191991635184, + "f1-score": 0.5197772743754077, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7443304673649471, + "recall": 0.7779698374809083, + "f1-score": 0.7098357783392529, + "support": 109341.0 + }, + "cross-entropy": 0.49273880116728924, + "roc-auc": 0.6982248815909875, + "score": 0.6982248815909875 + }, + "val": { + "0": { + "precision": 0.6767916685698625, + "recall": 0.9548266529191907, + "f1-score": 0.7921199647162599, + "support": 15518.0 + }, + "1": { + "precision": 0.7039695945945946, + "recall": 0.19066681916962142, + "f1-score": 0.3000630006300063, + "support": 8743.0 + }, + "accuracy": 0.6794443757470838, + "macro avg": { + "precision": 0.6903806315822285, + "recall": 0.5727467360444061, + "f1-score": 0.5460914826731331, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6865858488276521, + "recall": 0.6794443757470838, + "f1-score": 0.6147961100933624, + "support": 24261.0 + }, + "cross-entropy": 0.6764607634104041, + "roc-auc": 0.6244437709503304, + "score": 0.6244437709503304 + }, + "test": { + "0": { + "precision": 0.657613234183004, + "recall": 0.9772485521805933, + "f1-score": 0.7861846014880316, + "support": 16922.0 + }, + "1": { + "precision": 0.7056574923547401, + "recall": 0.09682156718766391, + "f1-score": 0.17027949451157642, + "support": 9533.0 + }, + "accuracy": 0.65998865998866, + "macro avg": { + "precision": 0.6816353632688721, + "recall": 0.5370350596841286, + "f1-score": 0.478232047999804, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6749258750127587, + "recall": 0.65998865998866, + "f1-score": 0.5642445756023182, + "support": 26455.0 + }, + "cross-entropy": 0.7397410477591574, + "roc-auc": 0.592244305956134, + "score": 0.592244305956134 + } + }, + "time": "0:03:40.866933", + "diversity": { + "train": { + "mean_centered_corr": -0.010312517173588276, + "mean_pairwise_disagreement": 0.02666120948639223, + "member_std": 0.029578423127532005 + }, + "val": { + "mean_centered_corr": -0.012083624489605427, + "mean_pairwise_disagreement": 0.09328301030061521, + "member_std": 0.06631172448396683 + }, + "test": { + "mean_centered_corr": -0.010351695120334625, + "mean_pairwise_disagreement": 0.058351141012431336, + "member_std": 0.06402193009853363 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ce05d4015 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,447 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7909716267830696, + "recall": 0.9643616578956747, + "f1-score": 0.8691029971468489, + "support": 84179.0 + }, + "1": { + "precision": 0.5528394693695037, + "recall": 0.1474048167872188, + "f1-score": 0.2327507765680399, + "support": 25162.0 + }, + "accuracy": 0.7763601942546712, + "macro avg": { + "precision": 0.6719055480762867, + "recall": 0.5558832373414467, + "f1-score": 0.5509268868574444, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.736171676674326, + "recall": 0.7763601942546712, + "f1-score": 0.7226630105525798, + "support": 109341.0 + }, + "cross-entropy": 0.5000760433868655, + "roc-auc": 0.6809180604528166, + "score": 0.6809180604528166 + }, + "val": { + "0": { + "precision": 0.7073092712054166, + "recall": 0.8549426472483568, + "f1-score": 0.7741502552881109, + "support": 15518.0 + }, + "1": { + "precision": 0.5910247093023255, + "recall": 0.37206908383849935, + "f1-score": 0.45665754193865377, + "support": 8743.0 + }, + "accuracy": 0.6809282387370678, + "macro avg": { + "precision": 0.649166990253871, + "recall": 0.613505865543428, + "f1-score": 0.6154038986133823, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6654034996082555, + "recall": 0.6809282387370678, + "f1-score": 0.6597345760986997, + "support": 24261.0 + }, + "cross-entropy": 0.6204112258943162, + "roc-auc": 0.6505096994576862, + "score": 0.6505096994576862 + }, + "test": { + "0": { + "precision": 0.6799947561615102, + "recall": 0.9195721545916559, + "f1-score": 0.7818419333768779, + "support": 16922.0 + }, + "1": { + "precision": 0.6188742649117894, + "recall": 0.2318262876324347, + "f1-score": 0.3373015873015873, + "support": 9533.0 + }, + "accuracy": 0.6717444717444717, + "macro avg": { + "precision": 0.6494345105366498, + "recall": 0.5756992211120453, + "f1-score": 0.5595717603392326, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6579701240283184, + "recall": 0.6717444717444717, + "f1-score": 0.6216528152844286, + "support": 26455.0 + }, + "cross-entropy": 0.6441552092220911, + "roc-auc": 0.6063775930816054, + "score": 0.6063775930816054 + } + }, + "time": "0:03:40.866933", + "diversity": { + "train": { + "mean_centered_corr": -0.010312517173588276, + "mean_pairwise_disagreement": 0.02666120948639223, + "member_std": 0.029578423127532005 + }, + "val": { + "mean_centered_corr": -0.012083624489605427, + "mean_pairwise_disagreement": 0.09328301030061521, + "member_std": 0.06631172448396683 + }, + "test": { + "mean_centered_corr": -0.010351695120334625, + "mean_pairwise_disagreement": 0.058351141012431336, + "member_std": 0.06402193009853363 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 22, + 24, + 24 + ], + "head_selection_time": "0:00:20.530825" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0da12dd14 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7835229427379597, + "recall": 0.9817888071846897, + "f1-score": 0.871522047464133, + "support": 84179.0 + }, + "1": { + "precision": 0.602952602952603, + "recall": 0.09252046737143312, + "f1-score": 0.1604244909209937, + "support": 25162.0 + }, + "accuracy": 0.7771467244674916, + "macro avg": { + "precision": 0.6932377728452814, + "recall": 0.5371546372780615, + "f1-score": 0.5159732691925634, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7419693545168977, + "recall": 0.7771467244674916, + "f1-score": 0.707881357167369, + "support": 109341.0 + }, + "cross-entropy": 0.49822654924283316, + "roc-auc": 0.6867900811541505, + "score": 0.6867900811541505 + }, + "val": { + "0": { + "precision": 0.6877843221759325, + "recall": 0.9255703054517335, + "f1-score": 0.7891541441169199, + "support": 15518.0 + }, + "1": { + "precision": 0.6580817051509769, + "recall": 0.2542605512981814, + "f1-score": 0.36680141902483293, + "support": 8743.0 + }, + "accuracy": 0.6836486542187049, + "macro avg": { + "precision": 0.6729330136634547, + "recall": 0.5899154283749575, + "f1-score": 0.5779777815708764, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6770803124216278, + "recall": 0.6836486542187049, + "f1-score": 0.6369497883409784, + "support": 24261.0 + }, + "cross-entropy": 0.6337455659020594, + "roc-auc": 0.6404461738890127, + "score": 0.6404461738890127 + }, + "test": { + "0": { + "precision": 0.6663787430075683, + "recall": 0.9573927431745657, + "f1-score": 0.785807828491051, + "support": 16922.0 + }, + "1": { + "precision": 0.6635557629491368, + "recall": 0.14916605475715933, + "f1-score": 0.24357656731757452, + "support": 9533.0 + }, + "accuracy": 0.6661500661500661, + "macro avg": { + "precision": 0.6649672529783526, + "recall": 0.5532793989658625, + "f1-score": 0.5146921979043128, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6653614884660062, + "recall": 0.6661500661500661, + "f1-score": 0.5904160079366473, + "support": 26455.0 + }, + "cross-entropy": 0.6685406788276543, + "roc-auc": 0.6034781946000056, + "score": 0.6034781946000056 + } + }, + "time": "0:01:00.788883", + "diversity": { + "train": { + "mean_centered_corr": -0.013087093830108643, + "mean_pairwise_disagreement": 0.025822423398123243, + "member_std": 0.02709772437810898 + }, + "val": { + "mean_centered_corr": -0.004288136959075928, + "mean_pairwise_disagreement": 0.08692423855623854, + "member_std": 0.060995232313871384 + }, + "test": { + "mean_centered_corr": 0.0036114430986344814, + "mean_pairwise_disagreement": 0.05469726437468373, + "member_std": 0.05942297354340553 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 14 + ], + "head_selection_time": "0:00:04.762789" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..bf2cd1131 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0.toml @@ -0,0 +1,33 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..9b9138da9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0/report.json @@ -0,0 +1,215 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7862557522292872, + "recall": 0.9783200085532021, + "f1-score": 0.8718353174078054, + "support": 84179.0 + }, + "1": { + "precision": 0.6031746031746031, + "recall": 0.11024560845719736, + "f1-score": 0.18641846712140048, + "support": 25162.0 + }, + "accuracy": 0.7785551622904492, + "macro avg": { + "precision": 0.6947151777019451, + "recall": 0.5442828085051997, + "f1-score": 0.529126892264603, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441243662668947, + "recall": 0.7785551622904492, + "f1-score": 0.7141043767093801, + "support": 109341.0 + }, + "cross-entropy": 0.4922807370219411, + "roc-auc": 0.6989988975549913, + "score": 0.6989988975549913 + }, + "val": { + "0": { + "precision": 0.6750500272876114, + "recall": 0.9565021265627014, + "f1-score": 0.7915000266624007, + "support": 15518.0 + }, + "1": { + "precision": 0.7030356357237132, + "recall": 0.1827747912615807, + "f1-score": 0.29012345679012347, + "support": 8743.0 + }, + "accuracy": 0.6776719838423808, + "macro avg": { + "precision": 0.6890428315056623, + "recall": 0.569638458912141, + "f1-score": 0.5408117417262621, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6851352741676592, + "recall": 0.6776719838423808, + "f1-score": 0.6108176413364323, + "support": 24261.0 + }, + "cross-entropy": 0.6872792555776136, + "roc-auc": 0.6227054517511603, + "score": 0.6227054517511603 + }, + "test": { + "0": { + "precision": 0.656363924552227, + "recall": 0.9788441082614349, + "f1-score": 0.7858057782627259, + "support": 16922.0 + }, + "1": { + "precision": 0.7063166529942576, + "recall": 0.09031784328123361, + "f1-score": 0.16015625, + "support": 9533.0 + }, + "accuracy": 0.6586656586656586, + "macro avg": { + "precision": 0.6813402887732423, + "recall": 0.5345809757713342, + "f1-score": 0.47298101413136295, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6743642783695727, + "recall": 0.6586656586656586, + "f1-score": 0.5603543719906198, + "support": 26455.0 + }, + "cross-entropy": 0.756433532699498, + "roc-auc": 0.591151981311678, + "score": 0.591151981311678 + } + }, + "time": "0:01:00.788883", + "diversity": { + "train": { + "mean_centered_corr": -0.013087093830108643, + "mean_pairwise_disagreement": 0.025822423398123243, + "member_std": 0.02709772437810898 + }, + "val": { + "mean_centered_corr": -0.004288136959075928, + "mean_pairwise_disagreement": 0.08692423855623854, + "member_std": 0.060995232313871384 + }, + "test": { + "mean_centered_corr": 0.0036114430986344814, + "mean_pairwise_disagreement": 0.05469726437468373, + "member_std": 0.05942297354340553 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5f26c2628 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,227 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7829606930730526, + "recall": 0.9834281709214887, + "f1-score": 0.8718189046448073, + "support": 84179.0 + }, + "1": { + "precision": 0.6134663341645885, + "recall": 0.08798982592798664, + "f1-score": 0.15390497375829829, + "support": 25162.0 + }, + "accuracy": 0.7773662212710694, + "macro avg": { + "precision": 0.6982135136188206, + "recall": 0.5357089984247376, + "f1-score": 0.5128619392015528, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439559550621073, + "recall": 0.7773662212710694, + "f1-score": 0.7066096022882681, + "support": 109341.0 + }, + "cross-entropy": 0.4977984361309486, + "roc-auc": 0.6854164410903827, + "score": 0.6854164410903827 + }, + "val": { + "0": { + "precision": 0.6785484611851171, + "recall": 0.9519267946900374, + "f1-score": 0.7923192447972538, + "support": 15518.0 + }, + "1": { + "precision": 0.7005218787635488, + "recall": 0.19958824202218917, + "f1-score": 0.31066405554566495, + "support": 8743.0 + }, + "accuracy": 0.6808045834879024, + "macro avg": { + "precision": 0.689535169974333, + "recall": 0.5757575183561132, + "f1-score": 0.5514916501714594, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6864670791270086, + "recall": 0.6808045834879024, + "f1-score": 0.6187439049668, + "support": 24261.0 + }, + "cross-entropy": 0.6384330519984234, + "roc-auc": 0.6413971749638401, + "score": 0.6413971749638401 + }, + "test": { + "0": { + "precision": 0.659637469489016, + "recall": 0.9741756293582319, + "f1-score": 0.7866294467110443, + "support": 16922.0 + }, + "1": { + "precision": 0.7015027322404371, + "recall": 0.1077310395468373, + "f1-score": 0.18677821223970173, + "support": 9533.0 + }, + "accuracy": 0.661954261954262, + "macro avg": { + "precision": 0.6805701008647266, + "recall": 0.5409533344525346, + "f1-score": 0.486703829475373, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6747235231578611, + "recall": 0.661954261954262, + "f1-score": 0.5704743978274568, + "support": 26455.0 + }, + "cross-entropy": 0.6791357747346614, + "roc-auc": 0.6040305589800323, + "score": 0.6040305589800323 + } + }, + "time": "0:01:00.788883", + "diversity": { + "train": { + "mean_centered_corr": -0.013087093830108643, + "mean_pairwise_disagreement": 0.025822423398123243, + "member_std": 0.02709772437810898 + }, + "val": { + "mean_centered_corr": -0.004288136959075928, + "mean_pairwise_disagreement": 0.08692423855623854, + "member_std": 0.060995232313871384 + }, + "test": { + "mean_centered_corr": 0.0036114430986344814, + "mean_pairwise_disagreement": 0.05469726437468373, + "member_std": 0.05942297354340553 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 14, + 0, + 25, + 25, + 0, + 25, + 0, + 25, + 0 + ], + "head_selection_time": "0:00:48.737044" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..480f3ae3e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-best-head-evaluation/0/report.json @@ -0,0 +1,390 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7849017777184993, + "recall": 0.9787239097637178, + "f1-score": 0.8711623357741415, + "support": 84179.0 + }, + "1": { + "precision": 0.5906285714285714, + "recall": 0.10269453938478658, + "f1-score": 0.17496699055422013, + "support": 25162.0 + }, + "accuracy": 0.7771284330671935, + "macro avg": { + "precision": 0.6877651745735354, + "recall": 0.5407092245742522, + "f1-score": 0.5230646631641808, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7401948295776631, + "recall": 0.7771284330671935, + "f1-score": 0.7109510035527089, + "support": 109341.0 + }, + "cross-entropy": 0.49898712269253065, + "roc-auc": 0.681041160411764, + "score": 0.681041160411764 + }, + "val": { + "0": { + "precision": 0.6882634613505326, + "recall": 0.9077200670189457, + "f1-score": 0.7829035126722989, + "support": 15518.0 + }, + "1": { + "precision": 0.6226613965744401, + "recall": 0.27027336154638, + "f1-score": 0.3769341202743659, + "support": 8743.0 + }, + "accuracy": 0.6780017311734883, + "macro avg": { + "precision": 0.6554624289624864, + "recall": 0.5889967142826629, + "f1-score": 0.5799188164733324, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6646222737516135, + "recall": 0.6780017311734883, + "f1-score": 0.6366032613333134, + "support": 24261.0 + }, + "cross-entropy": 0.630002257594146, + "roc-auc": 0.640920668337369, + "score": 0.640920668337369 + }, + "test": { + "0": { + "precision": 0.668389645318541, + "recall": 0.9399007209549699, + "f1-score": 0.7812269757846653, + "support": 16922.0 + }, + "1": { + "precision": 0.6175253854832644, + "recall": 0.17224378474771845, + "f1-score": 0.2693569553805774, + "support": 9533.0 + }, + "accuracy": 0.6632772632772633, + "macro avg": { + "precision": 0.6429575154009026, + "recall": 0.5560722528513442, + "f1-score": 0.5252919655826214, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6500608232051526, + "recall": 0.6632772632772633, + "f1-score": 0.5967757603428898, + "support": 26455.0 + }, + "cross-entropy": 0.6609475697303346, + "roc-auc": 0.6022998594088651, + "score": 0.6022998594088651 + } + }, + "time": "0:01:10.756545", + "diversity": { + "train": { + "mean_centered_corr": -0.01448837574571371, + "mean_pairwise_disagreement": 0.025696983482570507, + "member_std": 0.02678096853196621 + }, + "val": { + "mean_centered_corr": -0.006191454362124205, + "mean_pairwise_disagreement": 0.0848030690435067, + "member_std": 0.060151606798172 + }, + "test": { + "mean_centered_corr": 2.267197305627633e-05, + "mean_pairwise_disagreement": 0.053483014370111145, + "member_std": 0.05869301036000252 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:04.815166" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..6d8c4a5f0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..87a7db82c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,386 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7863576019789309, + "recall": 0.97808241960584, + "f1-score": 0.8718035598945373, + "support": 84179.0 + }, + "1": { + "precision": 0.6021992238033635, + "recall": 0.11100071536443844, + "f1-score": 0.1874496644295302, + "support": 25162.0 + }, + "accuracy": 0.7785460165903001, + "macro avg": { + "precision": 0.6942784128911472, + "recall": 0.5445415674851392, + "f1-score": 0.5296266121620338, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.743978319626898, + "recall": 0.7785460165903001, + "f1-score": 0.7143172307253279, + "support": 109341.0 + }, + "cross-entropy": 0.4923443699583362, + "roc-auc": 0.6988661347453451, + "score": 0.6988661347453451 + }, + "val": { + "0": { + "precision": 0.6754186385147434, + "recall": 0.9565021265627014, + "f1-score": 0.791753347202219, + "support": 15518.0 + }, + "1": { + "precision": 0.7045951859956237, + "recall": 0.18414731785428343, + "f1-score": 0.2919840406238665, + "support": 8743.0 + }, + "accuracy": 0.6781666048390421, + "macro avg": { + "precision": 0.6900069122551835, + "recall": 0.5703247222084924, + "f1-score": 0.5418686939130428, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6859330672120492, + "recall": 0.6781666048390421, + "f1-score": 0.6116501755516467, + "support": 24261.0 + }, + "cross-entropy": 0.6863324565732282, + "roc-auc": 0.6228570653182646, + "score": 0.6228570653182646 + }, + "test": { + "0": { + "precision": 0.6563998890078091, + "recall": 0.9785486349131308, + "f1-score": 0.7857363164013381, + "support": 16922.0 + }, + "1": { + "precision": 0.7043973941368078, + "recall": 0.09073743837197105, + "f1-score": 0.16076572809218473, + "support": 9533.0 + }, + "accuracy": 0.6586278586278587, + "macro avg": { + "precision": 0.6803986415723084, + "recall": 0.534643036642551, + "f1-score": 0.4732510222467614, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6736956824833239, + "recall": 0.6586278586278587, + "f1-score": 0.5605295646209125, + "support": 26455.0 + }, + "cross-entropy": 0.7549517436640245, + "roc-auc": 0.5912083608375949, + "score": 0.5912083608375949 + } + }, + "time": "0:01:10.756545", + "diversity": { + "train": { + "mean_centered_corr": -0.01448837574571371, + "mean_pairwise_disagreement": 0.025696983482570507, + "member_std": 0.02678096853196621 + }, + "val": { + "mean_centered_corr": -0.006191454362124205, + "mean_pairwise_disagreement": 0.0848030690435067, + "member_std": 0.060151606798172 + }, + "test": { + "mean_centered_corr": 2.267197305627633e-05, + "mean_pairwise_disagreement": 0.053483014370111145, + "member_std": 0.05869301036000252 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1066b9b8d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/cf_fisd_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,391 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7827204839776917, + "recall": 0.9836657598688509, + "f1-score": 0.871763287694308, + "support": 84179.0 + }, + "1": { + "precision": 0.6127851309490284, + "recall": 0.0864796121135045, + "f1-score": 0.15156897572528122, + "support": 25162.0 + }, + "accuracy": 0.777201598668386, + "macro avg": { + "precision": 0.6977528074633601, + "recall": 0.5350726859911776, + "f1-score": 0.5116661317097946, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436142625885768, + "recall": 0.777201598668386, + "f1-score": 0.7060292146771904, + "support": 109341.0 + }, + "cross-entropy": 0.49797948962611205, + "roc-auc": 0.6847429264219673, + "score": 0.6847429264219673 + }, + "val": { + "0": { + "precision": 0.6772866480778901, + "recall": 0.9548266529191907, + "f1-score": 0.7924588848776575, + "support": 15518.0 + }, + "1": { + "precision": 0.7059563758389261, + "recall": 0.19249685462655838, + "f1-score": 0.3025074143974117, + "support": 8743.0 + }, + "accuracy": 0.6801038704092989, + "macro avg": { + "precision": 0.6916215119584082, + "recall": 0.5736617537728745, + "f1-score": 0.5474831496375345, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6876184328276835, + "recall": 0.6801038704092989, + "f1-score": 0.6158937924903367, + "support": 24261.0 + }, + "cross-entropy": 0.6393924934998982, + "roc-auc": 0.6418041361448852, + "score": 0.6418041361448852 + }, + "test": { + "0": { + "precision": 0.6587311081867847, + "recall": 0.9761848481266989, + "f1-score": 0.7866377770899308, + "support": 16922.0 + }, + "1": { + "precision": 0.7075471698113207, + "recall": 0.1022763033672506, + "f1-score": 0.17871872422326093, + "support": 9533.0 + }, + "accuracy": 0.6612738612738612, + "macro avg": { + "precision": 0.6831391389990527, + "recall": 0.5392305757469747, + "f1-score": 0.48267825065659586, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6763218666621845, + "recall": 0.6612738612738612, + "f1-score": 0.5675755079166946, + "support": 26455.0 + }, + "cross-entropy": 0.6805522467825859, + "roc-auc": 0.6039617691395597, + "score": 0.6039617691395597 + } + }, + "time": "0:01:10.756545", + "diversity": { + "train": { + "mean_centered_corr": -0.01448837574571371, + "mean_pairwise_disagreement": 0.025696983482570507, + "member_std": 0.02678096853196621 + }, + "val": { + "mean_centered_corr": -0.006191454362124205, + "mean_pairwise_disagreement": 0.0848030690435067, + "member_std": 0.060151606798172 + }, + "test": { + "mean_centered_corr": 2.267197305627633e-05, + "mean_pairwise_disagreement": 0.053483014370111145, + "member_std": 0.05869301036000252 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 25 + ], + "head_selection_time": "0:00:15.897517" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..da99ccee1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7843107416636549, + "recall": 0.9796267477636941, + "f1-score": 0.8711553393442882, + "support": 84179.0 + }, + "1": { + "precision": 0.5915694212907835, + "recall": 0.09872029250457039, + "f1-score": 0.1692040461837131, + "support": 25162.0 + }, + "accuracy": 0.7769089362636157, + "macro avg": { + "precision": 0.6879400814772192, + "recall": 0.5391735201341322, + "f1-score": 0.5201796927640007, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7399563174017385, + "recall": 0.7769089362636157, + "f1-score": 0.709619424742205, + "support": 109341.0 + }, + "cross-entropy": 0.4992874898761024, + "roc-auc": 0.680286897888579, + "score": 0.680286897888579 + }, + "val": { + "0": { + "precision": 0.6876976212482366, + "recall": 0.9110065730119861, + "f1-score": 0.7837560637560638, + "support": 15518.0 + }, + "1": { + "precision": 0.6271598272138229, + "recall": 0.2656982729040375, + "f1-score": 0.3732626335663212, + "support": 8743.0 + }, + "accuracy": 0.6784551337537612, + "macro avg": { + "precision": 0.6574287242310297, + "recall": 0.5883524229580118, + "f1-score": 0.5785093486611925, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6658814581369519, + "recall": 0.6784551337537612, + "f1-score": 0.6358254730900188, + "support": 24261.0 + }, + "cross-entropy": 0.6313069278816857, + "roc-auc": 0.6406711951042247, + "score": 0.6406711951042247 + }, + "test": { + "0": { + "precision": 0.6672369189685388, + "recall": 0.9449828625457983, + "f1-score": 0.7821854822930934, + "support": 16922.0 + }, + "1": { + "precision": 0.6259541984732825, + "recall": 0.16343228784223224, + "f1-score": 0.2591914822824821, + "support": 9533.0 + }, + "accuracy": 0.6633528633528634, + "macro avg": { + "precision": 0.6465955587209107, + "recall": 0.5542075751940153, + "f1-score": 0.5206884822877877, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6523607830970105, + "recall": 0.6633528633528634, + "f1-score": 0.5937257657139531, + "support": 26455.0 + }, + "cross-entropy": 0.6637267283308612, + "roc-auc": 0.6023965414622969, + "score": 0.6023965414622969 + } + }, + "time": "0:01:33.770385", + "diversity": { + "train": { + "mean_centered_corr": -0.013189224526286125, + "mean_pairwise_disagreement": 0.02590141569537856, + "member_std": 0.027156289666891098 + }, + "val": { + "mean_centered_corr": -0.003924679942429066, + "mean_pairwise_disagreement": 0.08662631915552772, + "member_std": 0.060857683420181274 + }, + "test": { + "mean_centered_corr": 0.004035224672406912, + "mean_pairwise_disagreement": 0.054524420653452914, + "member_std": 0.059433531016111374 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:04.761324" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/0.toml new file mode 100644 index 000000000..6c0027518 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/0/report.json new file mode 100644 index 000000000..74020b221 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.786244164176397, + "recall": 0.9782962496584658, + "f1-score": 0.8718187592631802, + "support": 84179.0 + }, + "1": { + "precision": 0.6028260869565217, + "recall": 0.1102058659883952, + "f1-score": 0.18634500369598817, + "support": 25162.0 + }, + "accuracy": 0.7785277251900019, + "macro avg": { + "precision": 0.6945351255664594, + "recall": 0.5442510578234305, + "f1-score": 0.5290818814795841, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7440352429208157, + "recall": 0.7785277251900019, + "f1-score": 0.7140747232878215, + "support": 109341.0 + }, + "cross-entropy": 0.49232123937555056, + "roc-auc": 0.6989090994233629, + "score": 0.6989090994233629 + }, + "val": { + "0": { + "precision": 0.6752969013059107, + "recall": 0.956373243974739, + "f1-score": 0.7916255500733431, + "support": 15518.0 + }, + "1": { + "precision": 0.7035901926444834, + "recall": 0.18380418620610775, + "f1-score": 0.29146640065294277, + "support": 8743.0 + }, + "accuracy": 0.6779605127570999, + "macro avg": { + "precision": 0.689443546975197, + "recall": 0.5700887150904234, + "f1-score": 0.541545975363143, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6854930286779539, + "recall": 0.6779605127570999, + "f1-score": 0.611381889738544, + "support": 24261.0 + }, + "cross-entropy": 0.6870363427483223, + "roc-auc": 0.6227207494642631, + "score": 0.6227207494642631 + }, + "test": { + "0": { + "precision": 0.6564519326065411, + "recall": 0.9785486349131308, + "f1-score": 0.7857736019170997, + "support": 16922.0 + }, + "1": { + "precision": 0.7048780487804878, + "recall": 0.09094723591733976, + "f1-score": 0.1611074979095048, + "support": 9533.0 + }, + "accuracy": 0.6587034587034587, + "macro avg": { + "precision": 0.6806649906935145, + "recall": 0.5347479354152352, + "f1-score": 0.4734405499133022, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6739021751121632, + "recall": 0.6587034587034587, + "f1-score": 0.5606765703728017, + "support": 26455.0 + }, + "cross-entropy": 0.7560552620145878, + "roc-auc": 0.5911612394559284, + "score": 0.5911612394559284 + } + }, + "time": "0:01:33.770385", + "diversity": { + "train": { + "mean_centered_corr": -0.013189224526286125, + "mean_pairwise_disagreement": 0.02590141569537856, + "member_std": 0.027156289666891098 + }, + "val": { + "mean_centered_corr": -0.003924679942429066, + "mean_pairwise_disagreement": 0.08662631915552772, + "member_std": 0.060857683420181274 + }, + "test": { + "mean_centered_corr": 0.004035224672406912, + "mean_pairwise_disagreement": 0.054524420653452914, + "member_std": 0.059433531016111374 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..67767979d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,230 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7830293972986266, + "recall": 0.9834400503688568, + "f1-score": 0.8718661632517654, + "support": 84179.0 + }, + "1": { + "precision": 0.6145977329278407, + "recall": 0.0883475081472061, + "f1-score": 0.1544876472427812, + "support": 25162.0 + }, + "accuracy": 0.7774576782725602, + "macro avg": { + "precision": 0.6988135651132337, + "recall": 0.5358937792580315, + "f1-score": 0.5131769052472732, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7442692109193387, + "recall": 0.7774576782725602, + "f1-score": 0.706780072765872, + "support": 109341.0 + }, + "cross-entropy": 0.4979356402925865, + "roc-auc": 0.6847042861139584, + "score": 0.6847042861139584 + }, + "val": { + "0": { + "precision": 0.6772467413674823, + "recall": 0.95424668127336, + "f1-score": 0.7922317631008747, + "support": 15518.0 + }, + "1": { + "precision": 0.7036727879799666, + "recall": 0.19283998627473406, + "f1-score": 0.30272017236735793, + "support": 8743.0 + }, + "accuracy": 0.6798565599109683, + "macro avg": { + "precision": 0.6904597646737245, + "recall": 0.573543333774047, + "f1-score": 0.5474759677341163, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6867699648757033, + "recall": 0.6798565599109683, + "f1-score": 0.6158251913279412, + "support": 24261.0 + }, + "cross-entropy": 0.6400161363311784, + "roc-auc": 0.6415752785241469, + "score": 0.6415752785241469 + }, + "test": { + "0": { + "precision": 0.6584811640422563, + "recall": 0.9761257534570382, + "f1-score": 0.7864403551788988, + "support": 16922.0 + }, + "1": { + "precision": 0.7051094890510949, + "recall": 0.10133221441309137, + "f1-score": 0.17719893607264056, + "support": 9533.0 + }, + "accuracy": 0.6608958608958609, + "macro avg": { + "precision": 0.6817953265466756, + "recall": 0.5387289839350649, + "f1-score": 0.4818196456257697, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6752835765279588, + "recall": 0.6608958608958609, + "f1-score": 0.5669015742928675, + "support": 26455.0 + }, + "cross-entropy": 0.6818542221067515, + "roc-auc": 0.6040898334194843, + "score": 0.6040898334194843 + } + }, + "time": "0:01:33.770385", + "diversity": { + "train": { + "mean_centered_corr": -0.013189224526286125, + "mean_pairwise_disagreement": 0.02590141569537856, + "member_std": 0.027156289666891098 + }, + "val": { + "mean_centered_corr": -0.003924679942429066, + "mean_pairwise_disagreement": 0.08662631915552772, + "member_std": 0.060857683420181274 + }, + "test": { + "mean_centered_corr": 0.004035224672406912, + "mean_pairwise_disagreement": 0.054524420653452914, + "member_std": 0.059433531016111374 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 25 + ], + "head_selection_time": "0:00:15.755461" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..276526ec8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-best-head-evaluation/0/report.json @@ -0,0 +1,255 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7919161090894863, + "recall": 0.9617125411325865, + "f1-score": 0.8685939905690236, + "support": 84179.0 + }, + "1": { + "precision": 0.5468859834106565, + "recall": 0.15459820364041013, + "f1-score": 0.24105344694035633, + "support": 25162.0 + }, + "accuracy": 0.77597607484841, + "macro avg": { + "precision": 0.6694010462500715, + "recall": 0.5581553723864984, + "f1-score": 0.5548237187546899, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7355287701925426, + "recall": 0.77597607484841, + "f1-score": 0.7241817832654089, + "support": 109341.0 + }, + "cross-entropy": 0.5019382114215071, + "roc-auc": 0.6778537614421274, + "score": 0.6778537614421274 + }, + "val": { + "0": { + "precision": 0.7093391589295467, + "recall": 0.8369635262276066, + "f1-score": 0.7678845926451461, + "support": 15518.0 + }, + "1": { + "precision": 0.5748613678373382, + "recall": 0.39128445613633767, + "f1-score": 0.46563223084252076, + "support": 8743.0 + }, + "accuracy": 0.6763529945179506, + "macro avg": { + "precision": 0.6421002633834425, + "recall": 0.6141239911819721, + "f1-score": 0.6167584117438334, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.660877045763594, + "recall": 0.6763529945179506, + "f1-score": 0.6589611187883243, + "support": 24261.0 + }, + "cross-entropy": 0.6229299585114317, + "roc-auc": 0.6501781765294031, + "score": 0.6501781765294031 + }, + "test": { + "0": { + "precision": 0.6828953229398663, + "recall": 0.9059803805696726, + "f1-score": 0.7787767956923702, + "support": 16922.0 + }, + "1": { + "precision": 0.6027465667915106, + "recall": 0.25322563726004405, + "f1-score": 0.3566257940611612, + "support": 9533.0 + }, + "accuracy": 0.6707616707616708, + "macro avg": { + "precision": 0.6428209448656885, + "recall": 0.5796030089148583, + "f1-score": 0.5677012948767657, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6540138981671476, + "recall": 0.6707616707616708, + "f1-score": 0.6266556277260004, + "support": 26455.0 + }, + "cross-entropy": 0.6442264400252198, + "roc-auc": 0.6048261425892079, + "score": 0.6048261425892079 + } + }, + "time": "0:01:07.311309", + "diversity": { + "train": { + "mean_centered_corr": -0.010898238979279995, + "mean_pairwise_disagreement": 0.026737251557793006, + "member_std": 0.030055008828639984 + }, + "val": { + "mean_centered_corr": -0.008359640836715698, + "mean_pairwise_disagreement": 0.09315727751030128, + "member_std": 0.06754488497972488 + }, + "test": { + "mean_centered_corr": -0.005901505704969168, + "mean_pairwise_disagreement": 0.05854524725492467, + "member_std": 0.06570815294981003 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24 + ], + "head_selection_time": "0:00:04.817110" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0.toml new file mode 100644 index 000000000..54bfca637 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..078b99182 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-evaluation/0/report.json @@ -0,0 +1,251 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7843021876334646, + "recall": 0.9816937716057449, + "f1-score": 0.8719664035791161, + "support": 84179.0 + }, + "1": { + "precision": 0.6124245472837022, + "recall": 0.09677291153326445, + "f1-score": 0.16713569908710274, + "support": 25162.0 + }, + "accuracy": 0.77805214878225, + "macro avg": { + "precision": 0.6983633674585834, + "recall": 0.5392333415695046, + "f1-score": 0.5195510513331094, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7447489991087509, + "recall": 0.77805214878225, + "f1-score": 0.7097678670152651, + "support": 109341.0 + }, + "cross-entropy": 0.4927182387508514, + "roc-auc": 0.698241002315497, + "score": 0.698241002315497 + }, + "val": { + "0": { + "precision": 0.6764142335766423, + "recall": 0.9554710658590024, + "f1-score": 0.792082910411881, + "support": 15518.0 + }, + "1": { + "precision": 0.7048269970098249, + "recall": 0.18872240649662586, + "f1-score": 0.2977264525442079, + "support": 8743.0 + }, + "accuracy": 0.6791558468323647, + "macro avg": { + "precision": 0.6906206152932336, + "recall": 0.5720967361778142, + "f1-score": 0.5449046814780444, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6866534154197781, + "recall": 0.6791558468323647, + "f1-score": 0.6139303812029833, + "support": 24261.0 + }, + "cross-entropy": 0.6745751820826075, + "roc-auc": 0.6250090898119413, + "score": 0.6250090898119413 + }, + "test": { + "0": { + "precision": 0.6571337782014617, + "recall": 0.9776622148682189, + "f1-score": 0.7859755807876859, + "support": 16922.0 + }, + "1": { + "precision": 0.7044566067240031, + "recall": 0.09451379418860799, + "f1-score": 0.16666666666666666, + "support": 9533.0 + }, + "accuracy": 0.6594216594216594, + "macro avg": { + "precision": 0.6807951924627325, + "recall": 0.5360880045284134, + "f1-score": 0.4763211237271763, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6741864534728805, + "recall": 0.6594216594216594, + "f1-score": 0.5628090006207732, + "support": 26455.0 + }, + "cross-entropy": 0.7367691279234707, + "roc-auc": 0.5927112610884333, + "score": 0.5927112610884333 + } + }, + "time": "0:01:07.311309", + "diversity": { + "train": { + "mean_centered_corr": -0.010898238979279995, + "mean_pairwise_disagreement": 0.026737251557793006, + "member_std": 0.030055008828639984 + }, + "val": { + "mean_centered_corr": -0.008359640836715698, + "mean_pairwise_disagreement": 0.09315727751030128, + "member_std": 0.06754488497972488 + }, + "test": { + "mean_centered_corr": -0.005901505704969168, + "mean_pairwise_disagreement": 0.05854524725492467, + "member_std": 0.06570815294981003 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..444970f1f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/mfb_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,256 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7904454637331882, + "recall": 0.9662742489219401, + "f1-score": 0.8695605693729521, + "support": 84179.0 + }, + "1": { + "precision": 0.5589560354202268, + "recall": 0.14299340275017883, + "f1-score": 0.22772872559258203, + "support": 25162.0 + }, + "accuracy": 0.776817479262125, + "macro avg": { + "precision": 0.6747007495767074, + "recall": 0.5546338258360595, + "f1-score": 0.5486446474827671, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7371741657277673, + "recall": 0.776817479262125, + "f1-score": 0.7218595893818996, + "support": 109341.0 + }, + "cross-entropy": 0.4989347278683794, + "roc-auc": 0.6834597818089504, + "score": 0.6834597818089504 + }, + "val": { + "0": { + "precision": 0.7030905642970632, + "recall": 0.8840056708338704, + "f1-score": 0.7832367466955951, + "support": 15518.0 + }, + "1": { + "precision": 0.6210526315789474, + "recall": 0.3374127873727554, + "f1-score": 0.43726376639739123, + "support": 8743.0 + }, + "accuracy": 0.6870285643625572, + "macro avg": { + "precision": 0.6620715979380053, + "recall": 0.6107092291033129, + "f1-score": 0.6102502565464931, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6735263399965609, + "recall": 0.6870285643625572, + "f1-score": 0.6585575592446575, + "support": 24261.0 + }, + "cross-entropy": 0.6200265526927166, + "roc-auc": 0.6506721330887921, + "score": 0.6506721330887921 + }, + "test": { + "0": { + "precision": 0.6765875561257216, + "recall": 0.9349958633731238, + "f1-score": 0.7850745528072047, + "support": 16922.0 + }, + "1": { + "precision": 0.6416938110749185, + "recall": 0.2066505821881884, + "f1-score": 0.312623978417837, + "support": 9533.0 + }, + "accuracy": 0.6725382725382726, + "macro avg": { + "precision": 0.6591406836003201, + "recall": 0.5708232227806561, + "f1-score": 0.5488492656125209, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6640136732465189, + "recall": 0.6725382725382726, + "f1-score": 0.6148280464509831, + "support": 26455.0 + }, + "cross-entropy": 0.6485088259427312, + "roc-auc": 0.6064667434006787, + "score": 0.6064667434006787 + } + }, + "time": "0:01:07.311309", + "diversity": { + "train": { + "mean_centered_corr": -0.010898238979279995, + "mean_pairwise_disagreement": 0.026737251557793006, + "member_std": 0.030055008828639984 + }, + "val": { + "mean_centered_corr": -0.008359640836715698, + "mean_pairwise_disagreement": 0.09315727751030128, + "member_std": 0.06754488497972488 + }, + "test": { + "mean_centered_corr": -0.005901505704969168, + "mean_pairwise_disagreement": 0.05854524725492467, + "member_std": 0.06570815294981003 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24, + 22 + ], + "head_selection_time": "0:00:15.861115" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f9259477f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-best-head-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7796513262723779, + "recall": 0.9881443115266277, + "f1-score": 0.8716030366376939, + "support": 84179.0 + }, + "1": { + "precision": 0.6235382874387023, + "recall": 0.06569430092997378, + "f1-score": 0.11886527882644807, + "support": 25162.0 + }, + "accuracy": 0.7758663264466211, + "macro avg": { + "precision": 0.7015948068555401, + "recall": 0.5269193062283007, + "f1-score": 0.49523415773207097, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7437259525961453, + "recall": 0.7758663264466211, + "f1-score": 0.6983799322025179, + "support": 109341.0 + }, + "cross-entropy": 0.500117320045548, + "roc-auc": 0.682915560586896, + "score": 0.682915560586896 + }, + "val": { + "0": { + "precision": 0.6736205183780367, + "recall": 0.9613352236112901, + "f1-score": 0.7921622769753611, + "support": 15518.0 + }, + "1": { + "precision": 0.7163120567375887, + "recall": 0.17328148232872012, + "f1-score": 0.279056916559219, + "support": 8743.0 + }, + "accuracy": 0.6773422365112732, + "macro avg": { + "precision": 0.6949662875578126, + "recall": 0.5673083529700051, + "f1-score": 0.53560959676729, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6890053796730189, + "recall": 0.6773422365112732, + "f1-score": 0.6072531567363632, + "support": 24261.0 + }, + "cross-entropy": 0.6460301333180669, + "roc-auc": 0.6394530055211662, + "score": 0.6394530055211662 + }, + "test": { + "0": { + "precision": 0.656934306569343, + "recall": 0.9786077295827916, + "f1-score": 0.7861381438404937, + "support": 16922.0 + }, + "1": { + "precision": 0.7097032878909383, + "recall": 0.09283541382565824, + "f1-score": 0.16419294990723562, + "support": 9533.0 + }, + "accuracy": 0.6594216594216594, + "macro avg": { + "precision": 0.6833187972301407, + "recall": 0.535721571704225, + "f1-score": 0.4751655468738647, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6759494908044128, + "recall": 0.6594216594216594, + "f1-score": 0.5620215861475908, + "support": 26455.0 + }, + "cross-entropy": 0.6825101151285496, + "roc-auc": 0.6014026252811646, + "score": 0.6014026252811646 + } + }, + "time": "0:01:49.861501", + "diversity": { + "train": { + "mean_centered_corr": -0.01670769229531288, + "mean_pairwise_disagreement": 0.024668442555119807, + "member_std": 0.02791455015540123 + }, + "val": { + "mean_centered_corr": 0.002592463046312332, + "mean_pairwise_disagreement": 0.06743399402465924, + "member_std": 0.05864376202225685 + }, + "test": { + "mean_centered_corr": 0.009284000843763351, + "mean_pairwise_disagreement": 0.0444435468629017, + "member_std": 0.060093436390161514 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.811023" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-evaluation/0.toml new file mode 100644 index 000000000..f6aef79c4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-evaluation/0/report.json new file mode 100644 index 000000000..03bee2ffe --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7814041960149158, + "recall": 0.9857803015003742, + "f1-score": 0.871774130006566, + "support": 84179.0 + }, + "1": { + "precision": 0.61939586645469, + "recall": 0.07741832922661156, + "f1-score": 0.13763380082665067, + "support": 25162.0 + }, + "accuracy": 0.7767443136609323, + "macro avg": { + "precision": 0.7004000312348029, + "recall": 0.5315993153634929, + "f1-score": 0.5047039654166083, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441221738238402, + "recall": 0.7767443136609323, + "f1-score": 0.7028307422304798, + "support": 109341.0 + }, + "cross-entropy": 0.4956439792677601, + "roc-auc": 0.6916789260357137, + "score": 0.6916789260357137 + }, + "val": { + "0": { + "precision": 0.6749659863945578, + "recall": 0.9590797783219487, + "f1-score": 0.7923232538330494, + "support": 15518.0 + }, + "1": { + "precision": 0.7127996381727725, + "recall": 0.18025849250829235, + "f1-score": 0.28774876757348916, + "support": 8743.0 + }, + "accuracy": 0.6784139153373727, + "macro avg": { + "precision": 0.6938828122836651, + "recall": 0.5696691354151205, + "f1-score": 0.5400360107032692, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6886001984013561, + "recall": 0.6784139153373727, + "f1-score": 0.6104884270176941, + "support": 24261.0 + }, + "cross-entropy": 0.6753608510436575, + "roc-auc": 0.6226365549199251, + "score": 0.6226365549199251 + }, + "test": { + "0": { + "precision": 0.6569933396764985, + "recall": 0.9793168656187212, + "f1-score": 0.7864091491481991, + "support": 16922.0 + }, + "1": { + "precision": 0.7156783103168156, + "recall": 0.0924158187349208, + "f1-score": 0.16369379412857674, + "support": 9533.0 + }, + "accuracy": 0.6597240597240597, + "macro avg": { + "precision": 0.6863358249966571, + "recall": 0.535866342176821, + "f1-score": 0.47505147163838796, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6781403374128109, + "recall": 0.6597240597240597, + "f1-score": 0.5620150656327177, + "support": 26455.0 + }, + "cross-entropy": 0.7367195601018909, + "roc-auc": 0.5911320299643263, + "score": 0.5911320299643263 + } + }, + "time": "0:01:49.861501", + "diversity": { + "train": { + "mean_centered_corr": -0.01670769229531288, + "mean_pairwise_disagreement": 0.024668442555119807, + "member_std": 0.02791455015540123 + }, + "val": { + "mean_centered_corr": 0.002592463046312332, + "mean_pairwise_disagreement": 0.06743399402465924, + "member_std": 0.05864376202225685 + }, + "test": { + "mean_centered_corr": 0.009284000843763351, + "mean_pairwise_disagreement": 0.0444435468629017, + "member_std": 0.060093436390161514 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..418bc48ef --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/ecom-offers/rla_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7796513262723779, + "recall": 0.9881443115266277, + "f1-score": 0.8716030366376939, + "support": 84179.0 + }, + "1": { + "precision": 0.6235382874387023, + "recall": 0.06569430092997378, + "f1-score": 0.11886527882644807, + "support": 25162.0 + }, + "accuracy": 0.7758663264466211, + "macro avg": { + "precision": 0.7015948068555401, + "recall": 0.5269193062283007, + "f1-score": 0.49523415773207097, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7437259525961453, + "recall": 0.7758663264466211, + "f1-score": 0.6983799322025179, + "support": 109341.0 + }, + "cross-entropy": 0.500117320045548, + "roc-auc": 0.682915560586896, + "score": 0.682915560586896 + }, + "val": { + "0": { + "precision": 0.6736205183780367, + "recall": 0.9613352236112901, + "f1-score": 0.7921622769753611, + "support": 15518.0 + }, + "1": { + "precision": 0.7163120567375887, + "recall": 0.17328148232872012, + "f1-score": 0.279056916559219, + "support": 8743.0 + }, + "accuracy": 0.6773422365112732, + "macro avg": { + "precision": 0.6949662875578126, + "recall": 0.5673083529700051, + "f1-score": 0.53560959676729, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6890053796730189, + "recall": 0.6773422365112732, + "f1-score": 0.6072531567363632, + "support": 24261.0 + }, + "cross-entropy": 0.6460301333180669, + "roc-auc": 0.6394530055211662, + "score": 0.6394530055211662 + }, + "test": { + "0": { + "precision": 0.656934306569343, + "recall": 0.9786077295827916, + "f1-score": 0.7861381438404937, + "support": 16922.0 + }, + "1": { + "precision": 0.7097032878909383, + "recall": 0.09283541382565824, + "f1-score": 0.16419294990723562, + "support": 9533.0 + }, + "accuracy": 0.6594216594216594, + "macro avg": { + "precision": 0.6833187972301407, + "recall": 0.535721571704225, + "f1-score": 0.4751655468738647, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6759494908044128, + "recall": 0.6594216594216594, + "f1-score": 0.5620215861475908, + "support": 26455.0 + }, + "cross-entropy": 0.6825101151285496, + "roc-auc": 0.6014026252811646, + "score": 0.6014026252811646 + } + }, + "time": "0:01:49.861501", + "diversity": { + "train": { + "mean_centered_corr": -0.01670769229531288, + "mean_pairwise_disagreement": 0.024668442555119807, + "member_std": 0.02791455015540123 + }, + "val": { + "mean_centered_corr": 0.002592463046312332, + "mean_pairwise_disagreement": 0.06743399402465924, + "member_std": 0.05864376202225685 + }, + "test": { + "mean_centered_corr": 0.009284000843763351, + "mean_pairwise_disagreement": 0.0444435468629017, + "member_std": 0.060093436390161514 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.087588" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..69cc42063 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-best-head-evaluation/0/report.json @@ -0,0 +1,680 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9513552811475944, + "recall": 0.9636150815239791, + "f1-score": 0.9574459372022485, + "support": 181971.0 + }, + "1": { + "precision": 0.8344915508449156, + "recall": 0.7882830763418263, + "f1-score": 0.8107294209075565, + "support": 42349.0 + }, + "accuracy": 0.9305144436519258, + "macro avg": { + "precision": 0.8929234159962549, + "recall": 0.8759490789329027, + "f1-score": 0.8840876790549025, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9292927717209353, + "recall": 0.9305144436519258, + "f1-score": 0.9297475699208473, + "support": 224320.0 + }, + "cross-entropy": 0.15873224362284377, + "roc-auc": 0.9722680689598284, + "score": 0.9722680689598284 + }, + "val": { + "0": { + "precision": 0.9436578686709782, + "recall": 0.9674451944328296, + "f1-score": 0.9554034922133081, + "support": 16741.0 + }, + "1": { + "precision": 0.8168067226890756, + "recall": 0.7153370621136297, + "f1-score": 0.7627118644067796, + "support": 3397.0 + }, + "accuracy": 0.9249180653490913, + "macro avg": { + "precision": 0.8802322956800269, + "recall": 0.8413911282732296, + "f1-score": 0.8590576783100439, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9222598478694823, + "recall": 0.9249180653490913, + "f1-score": 0.9228990995398164, + "support": 20138.0 + }, + "cross-entropy": 0.17990073731774503, + "roc-auc": 0.9569685349939213, + "score": 0.9569685349939213 + }, + "test": { + "0": { + "precision": 0.9437623162043353, + "recall": 0.9471362287974443, + "f1-score": 0.9454462624805436, + "support": 13147.0 + }, + "1": { + "precision": 0.7758787487907127, + "recall": 0.764294790343075, + "f1-score": 0.7700432069131061, + "support": 3148.0 + }, + "accuracy": 0.9118134397054312, + "macro avg": { + "precision": 0.859820532497524, + "recall": 0.8557155095702597, + "f1-score": 0.8577447346968249, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9113292097165732, + "recall": 0.9118134397054312, + "f1-score": 0.9115604804046742, + "support": 16295.0 + }, + "cross-entropy": 0.19410702240157, + "roc-auc": 0.9582203301945192, + "score": 0.9582203301945192 + } + }, + "time": "0:14:26.086332", + "diversity": { + "train": { + "mean_centered_corr": 0.09211483597755432, + "mean_pairwise_disagreement": 0.03578823167364594, + "member_std": 0.03491802513599396 + }, + "val": { + "mean_centered_corr": 0.1013437807559967, + "mean_pairwise_disagreement": 0.03309443068632885, + "member_std": 0.03697067126631737 + }, + "test": { + "mean_centered_corr": 0.12410959601402283, + "mean_pairwise_disagreement": 0.0401679220817785, + "member_std": 0.03913382440805435 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6 + ], + "head_selection_time": "0:00:01.132889" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..5b9249ca9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0.toml @@ -0,0 +1,75 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..6ef0bc1f3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/0/report.json @@ -0,0 +1,676 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9439755183585884, + "recall": 0.9814915563468904, + "f1-score": 0.9623680537968715, + "support": 181971.0 + }, + "1": { + "precision": 0.9040920351966284, + "recall": 0.7496989303171268, + "f1-score": 0.8196886375958485, + "support": 42349.0 + }, + "accuracy": 0.9377318116975749, + "macro avg": { + "precision": 0.9240337767776083, + "recall": 0.8655952433320087, + "f1-score": 0.89102834569636, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9364459818552635, + "recall": 0.9377318116975749, + "f1-score": 0.9354318439328508, + "support": 224320.0 + }, + "cross-entropy": 0.14691183931212085, + "roc-auc": 0.9764623137919726, + "score": 0.9764623137919726 + }, + "val": { + "0": { + "precision": 0.934692250754427, + "recall": 0.980586583836091, + "f1-score": 0.957089552238806, + "support": 16741.0 + }, + "1": { + "precision": 0.8737864077669902, + "recall": 0.6623491315866942, + "f1-score": 0.753516409912927, + "support": 3397.0 + }, + "accuracy": 0.9269043599165756, + "macro avg": { + "precision": 0.9042393292607086, + "recall": 0.8214678577113925, + "f1-score": 0.8553029810758666, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.924418283695716, + "recall": 0.9269043599165756, + "f1-score": 0.9227495996873605, + "support": 20138.0 + }, + "cross-entropy": 0.17471226841771598, + "roc-auc": 0.958474253988237, + "score": 0.958474253988237 + }, + "test": { + "0": { + "precision": 0.9308116741687237, + "recall": 0.9752034684718948, + "f1-score": 0.9524906207050258, + "support": 13147.0 + }, + "1": { + "precision": 0.8706862356207854, + "recall": 0.6972681067344345, + "f1-score": 0.7743870171106015, + "support": 3148.0 + }, + "accuracy": 0.9215096655415772, + "macro avg": { + "precision": 0.9007489548947545, + "recall": 0.8362357876031646, + "f1-score": 0.8634388189078137, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9191961552642187, + "recall": 0.9215096655415772, + "f1-score": 0.9180831249016966, + "support": 16295.0 + }, + "cross-entropy": 0.17874492250186327, + "roc-auc": 0.962487069534998, + "score": 0.962487069534998 + } + }, + "time": "0:14:26.086332", + "diversity": { + "train": { + "mean_centered_corr": 0.09211483597755432, + "mean_pairwise_disagreement": 0.03578823167364594, + "member_std": 0.03491802513599396 + }, + "val": { + "mean_centered_corr": 0.1013437807559967, + "mean_pairwise_disagreement": 0.03309443068632885, + "member_std": 0.03697067126631737 + }, + "test": { + "mean_centered_corr": 0.12410959601402283, + "mean_pairwise_disagreement": 0.0401679220817785, + "member_std": 0.03913382440805435 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5acbc3d79 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9442336152219873, + "recall": 0.9817443438789697, + "f1-score": 0.9626236963555881, + "support": 181971.0 + }, + "1": { + "precision": 0.9054100227790433, + "recall": 0.7508559824316985, + "f1-score": 0.8209219171539583, + "support": 42349.0 + }, + "accuracy": 0.9381553138373752, + "macro avg": { + "precision": 0.9248218190005153, + "recall": 0.8663001631553341, + "f1-score": 0.8917728067547732, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9369041737260607, + "recall": 0.9381553138373752, + "f1-score": 0.9358720529559367, + "support": 224320.0 + }, + "cross-entropy": 0.14721044455331747, + "roc-auc": 0.9763538308497102, + "score": 0.9763538308497102 + }, + "val": { + "0": { + "precision": 0.9343600818367811, + "recall": 0.9820799235410071, + "f1-score": 0.9576258846142645, + "support": 16741.0 + }, + "1": { + "precision": 0.8819826907946499, + "recall": 0.6599941124521637, + "f1-score": 0.7550092608183195, + "support": 3397.0 + }, + "accuracy": 0.9277485351077565, + "macro avg": { + "precision": 0.9081713863157155, + "recall": 0.8210370179965853, + "f1-score": 0.8563175727162919, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9255247457870185, + "recall": 0.9277485351077565, + "f1-score": 0.9234472834108269, + "support": 20138.0 + }, + "cross-entropy": 0.17362982513958533, + "roc-auc": 0.9592737644154757, + "score": 0.9592737644154757 + }, + "test": { + "0": { + "precision": 0.9306966618287373, + "recall": 0.975507720392485, + "f1-score": 0.9525754818583578, + "support": 13147.0 + }, + "1": { + "precision": 0.8719681908548708, + "recall": 0.6966327827191868, + "f1-score": 0.7745011478015186, + "support": 3148.0 + }, + "accuracy": 0.9216324025774778, + "macro avg": { + "precision": 0.9013324263418041, + "recall": 0.8360702515558359, + "f1-score": 0.8635383148299383, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.919351020427956, + "recall": 0.9216324025774778, + "f1-score": 0.9181736405812219, + "support": 16295.0 + }, + "cross-entropy": 0.17835468537288718, + "roc-auc": 0.9627526496640615, + "score": 0.9627526496640615 + } + }, + "time": "0:14:26.086332", + "diversity": { + "train": { + "mean_centered_corr": 0.09211483597755432, + "mean_pairwise_disagreement": 0.03578823167364594, + "member_std": 0.03491802513599396 + }, + "val": { + "mean_centered_corr": 0.1013437807559967, + "mean_pairwise_disagreement": 0.03309443068632885, + "member_std": 0.03697067126631737 + }, + "test": { + "mean_centered_corr": 0.12410959601402283, + "mean_pairwise_disagreement": 0.0401679220817785, + "member_std": 0.03913382440805435 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6, + 31, + 30, + 28, + 8, + 20, + 12, + 30, + 8, + 21, + 15 + ], + "head_selection_time": "0:00:14.217902" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..67ae69ad9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,262 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9509438009438009, + "recall": 0.9745069269279171, + "f1-score": 0.9625811844722908, + "support": 181971.0 + }, + "1": { + "precision": 0.8774048625792812, + "recall": 0.7839854542019883, + "f1-score": 0.8280686877252491, + "support": 42349.0 + }, + "accuracy": 0.9385386947218259, + "macro avg": { + "precision": 0.9141743317615411, + "recall": 0.8792461905649527, + "f1-score": 0.89532493609877, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9370605069851746, + "recall": 0.9385386947218259, + "f1-score": 0.9371867937592895, + "support": 224320.0 + }, + "cross-entropy": 0.14097183098455576, + "roc-auc": 0.9772540623630491, + "score": 0.9772540623630491 + }, + "val": { + "0": { + "precision": 0.9410365039753932, + "recall": 0.9685801326085658, + "f1-score": 0.9546096785588132, + "support": 16741.0 + }, + "1": { + "precision": 0.8190574475404196, + "recall": 0.7009125699146306, + "f1-score": 0.7553934010152284, + "support": 3397.0 + }, + "accuracy": 0.923428344423478, + "macro avg": { + "precision": 0.8800469757579064, + "recall": 0.8347463512615982, + "f1-score": 0.8550015397870208, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9204603367934683, + "recall": 0.923428344423478, + "f1-score": 0.9210046683882125, + "support": 20138.0 + }, + "cross-entropy": 0.181709302619563, + "roc-auc": 0.9554763470552776, + "score": 0.9554763470552776 + }, + "test": { + "0": { + "precision": 0.9398939744642724, + "recall": 0.9574807940975127, + "f1-score": 0.9486058779201205, + "support": 13147.0 + }, + "1": { + "precision": 0.8073742246726395, + "recall": 0.7442820838627701, + "f1-score": 0.7745454545454545, + "support": 3148.0 + }, + "accuracy": 0.9162933415158024, + "macro avg": { + "precision": 0.873634099568456, + "recall": 0.8508814389801413, + "f1-score": 0.8615756662327876, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9142927365174138, + "recall": 0.9162933415158024, + "f1-score": 0.914979476399197, + "support": 16295.0 + }, + "cross-entropy": 0.18955508005758365, + "roc-auc": 0.959416932798502, + "score": 0.959416932798502 + } + }, + "time": "0:02:48.403690", + "diversity": { + "train": { + "mean_centered_corr": 0.05291927233338356, + "mean_pairwise_disagreement": 0.03400679940235148, + "member_std": 0.03158120810985565 + }, + "val": { + "mean_centered_corr": 0.06288693100214005, + "mean_pairwise_disagreement": 0.03244768356405319, + "member_std": 0.0349886491894722 + }, + "test": { + "mean_centered_corr": 0.07999585568904877, + "mean_pairwise_disagreement": 0.041502439893495925, + "member_std": 0.0369756743311882 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.916152" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..d85b237a3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0.toml @@ -0,0 +1,34 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..cd297e4bc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0/report.json @@ -0,0 +1,258 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9482836172363248, + "recall": 0.984772298882789, + "f1-score": 0.966183574879227, + "support": 181971.0 + }, + "1": { + "precision": 0.9216057939853453, + "recall": 0.769227136414083, + "f1-score": 0.8385502471169687, + "support": 42349.0 + }, + "accuracy": 0.9440798858773182, + "macro avg": { + "precision": 0.9349447056108351, + "recall": 0.8769997176484361, + "f1-score": 0.9023669109980978, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.943247155320955, + "recall": 0.9440798858773182, + "f1-score": 0.9420878910462924, + "support": 224320.0 + }, + "cross-entropy": 0.13271224543526663, + "roc-auc": 0.9813876825616359, + "score": 0.9813876825616359 + }, + "val": { + "0": { + "precision": 0.9351403678606002, + "recall": 0.9809449853652709, + "f1-score": 0.9574951897848522, + "support": 16741.0 + }, + "1": { + "precision": 0.8762126503686457, + "recall": 0.6647041507212246, + "f1-score": 0.7559424171409441, + "support": 3397.0 + }, + "accuracy": 0.9275995630151952, + "macro avg": { + "precision": 0.9056765091146229, + "recall": 0.8228245680432478, + "f1-score": 0.8567188034628981, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9252000830100605, + "recall": 0.9275995630151952, + "f1-score": 0.9234960454472141, + "support": 20138.0 + }, + "cross-entropy": 0.17351272311805546, + "roc-auc": 0.9583121450834431, + "score": 0.9583121450834431 + }, + "test": { + "0": { + "precision": 0.9321849105974706, + "recall": 0.975507720392485, + "f1-score": 0.9533543950938487, + "support": 13147.0 + }, + "1": { + "precision": 0.8730784391013008, + "recall": 0.7036213468869124, + "f1-score": 0.7792436235708003, + "support": 3148.0 + }, + "accuracy": 0.9229825099723842, + "macro avg": { + "precision": 0.9026316748493857, + "recall": 0.8395645336396986, + "f1-score": 0.8662990093323245, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9207662439960627, + "recall": 0.9229825099723842, + "f1-score": 0.9197182669100772, + "support": 16295.0 + }, + "cross-entropy": 0.17696141048282132, + "roc-auc": 0.9626260391126089, + "score": 0.9626260391126089 + } + }, + "time": "0:02:48.403690", + "diversity": { + "train": { + "mean_centered_corr": 0.05291927233338356, + "mean_pairwise_disagreement": 0.03400679940235148, + "member_std": 0.03158120810985565 + }, + "val": { + "mean_centered_corr": 0.06288693100214005, + "mean_pairwise_disagreement": 0.03244768356405319, + "member_std": 0.0349886491894722 + }, + "test": { + "mean_centered_corr": 0.07999585568904877, + "mean_pairwise_disagreement": 0.041502439893495925, + "member_std": 0.0369756743311882 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0b339388c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9453926712602988, + "recall": 0.9862120887394146, + "f1-score": 0.9653710741556594, + "support": 181971.0 + }, + "1": { + "precision": 0.9272584947234142, + "recall": 0.7552244444969185, + "f1-score": 0.8324462201168646, + "support": 42349.0 + }, + "accuracy": 0.9426043152639088, + "macro avg": { + "precision": 0.9363255829918564, + "recall": 0.8707182666181665, + "f1-score": 0.898908647136262, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9419691502092978, + "recall": 0.9426043152639088, + "f1-score": 0.9402764118710262, + "support": 224320.0 + }, + "cross-entropy": 0.13432360872538882, + "roc-auc": 0.9807020207732833, + "score": 0.9807020207732833 + }, + "val": { + "0": { + "precision": 0.9332728098048116, + "recall": 0.9824980586583836, + "f1-score": 0.9572530190600902, + "support": 16741.0 + }, + "1": { + "precision": 0.8834526650755767, + "recall": 0.6538121872240212, + "f1-score": 0.7514802909829132, + "support": 3397.0 + }, + "accuracy": 0.927053332009137, + "macro avg": { + "precision": 0.9083627374401941, + "recall": 0.8181551229412024, + "f1-score": 0.8543666550215017, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9248688455757318, + "recall": 0.927053332009137, + "f1-score": 0.9225420270411127, + "support": 20138.0 + }, + "cross-entropy": 0.17301902153958362, + "roc-auc": 0.9589474980445032, + "score": 0.9589474980445032 + }, + "test": { + "0": { + "precision": 0.9310144506571781, + "recall": 0.9752034684718948, + "f1-score": 0.952596775391931, + "support": 13147.0 + }, + "1": { + "precision": 0.8708399366085579, + "recall": 0.6982210927573063, + "f1-score": 0.7750352609308886, + "support": 3148.0 + }, + "accuracy": 0.921693771095428, + "macro avg": { + "precision": 0.900927193632868, + "recall": 0.8367122806146006, + "f1-score": 0.8638160181614098, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9193894509502093, + "recall": 0.921693771095428, + "f1-score": 0.9182940047553332, + "support": 16295.0 + }, + "cross-entropy": 0.17739591453615167, + "roc-auc": 0.9625170670540112, + "score": 0.9625170670540112 + } + }, + "time": "0:02:48.403690", + "diversity": { + "train": { + "mean_centered_corr": 0.05291927233338356, + "mean_pairwise_disagreement": 0.03400679940235148, + "member_std": 0.03158120810985565 + }, + "val": { + "mean_centered_corr": 0.06288693100214005, + "mean_pairwise_disagreement": 0.03244768356405319, + "member_std": 0.0349886491894722 + }, + "test": { + "mean_centered_corr": 0.07999585568904877, + "mean_pairwise_disagreement": 0.041502439893495925, + "member_std": 0.0369756743311882 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 6, + 9, + 15, + 28, + 3, + 23, + 26, + 2, + 20, + 13, + 6, + 15, + 5 + ], + "head_selection_time": "0:00:13.989049" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a89b4d488 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-best-head-evaluation/0/report.json @@ -0,0 +1,621 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2420, + "metrics": { + "train": { + "0": { + "precision": 0.9511390416339356, + "recall": 0.9648020838485253, + "f1-score": 0.9579218455024607, + "support": 181971.0 + }, + "1": { + "precision": 0.8388070970177426, + "recall": 0.7870315709934119, + "f1-score": 0.8120949271477998, + "support": 42349.0 + }, + "accuracy": 0.9312410841654779, + "macro avg": { + "precision": 0.8949730693258391, + "recall": 0.8759168274209685, + "f1-score": 0.8850083863251303, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9299320804956013, + "recall": 0.9312410841654779, + "f1-score": 0.9303914239377251, + "support": 224320.0 + }, + "cross-entropy": 0.15615645016500299, + "roc-auc": 0.9724991426058968, + "score": 0.9724991426058968 + }, + "val": { + "0": { + "precision": 0.9410138248847926, + "recall": 0.9758078967803596, + "f1-score": 0.958095070525791, + "support": 16741.0 + }, + "1": { + "precision": 0.8542116630669546, + "recall": 0.6985575507801001, + "f1-score": 0.768582995951417, + "support": 3397.0 + }, + "accuracy": 0.9290396265766213, + "macro avg": { + "precision": 0.8976127439758737, + "recall": 0.8371827237802298, + "f1-score": 0.863339033238604, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9263715096749804, + "recall": 0.9290396265766213, + "f1-score": 0.9261270241791256, + "support": 20138.0 + }, + "cross-entropy": 0.17528768630111824, + "roc-auc": 0.957813544584969, + "score": 0.957813544584969 + }, + "test": { + "0": { + "precision": 0.9342733348020856, + "recall": 0.9676732334372861, + "f1-score": 0.9506800179345389, + "support": 13147.0 + }, + "1": { + "precision": 0.8412994772218073, + "recall": 0.71569250317662, + "f1-score": 0.7734294541709578, + "support": 3148.0 + }, + "accuracy": 0.9189935563056152, + "macro avg": { + "precision": 0.8877864060119465, + "recall": 0.841682868306953, + "f1-score": 0.8620547360527484, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9163118924171383, + "recall": 0.9189935563056152, + "f1-score": 0.9164373192706695, + "support": 16295.0 + }, + "cross-entropy": 0.1803789882797406, + "roc-auc": 0.961490313471295, + "score": 0.961490313471295 + } + }, + "time": "0:03:46.650441", + "diversity": { + "train": { + "mean_centered_corr": 0.0506281740963459, + "mean_pairwise_disagreement": 0.03327413710540242, + "member_std": 0.03079444356262684 + }, + "val": { + "mean_centered_corr": 0.056434109807014465, + "mean_pairwise_disagreement": 0.030276074921749604, + "member_std": 0.03229342773556709 + }, + "test": { + "mean_centered_corr": 0.06734751909971237, + "mean_pairwise_disagreement": 0.037917825574834946, + "member_std": 0.03452758118510246 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.860463" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..e655e5952 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0.toml @@ -0,0 +1,45 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..5caf5c656 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,617 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2420, + "metrics": { + "train": { + "0": { + "precision": 0.9480090078156047, + "recall": 0.9831951244978595, + "f1-score": 0.965281524483674, + "support": 181971.0 + }, + "1": { + "precision": 0.9140890574518893, + "recall": 0.7683062173841176, + "f1-score": 0.8348814533511238, + "support": 42349.0 + }, + "accuracy": 0.942626604850214, + "macro avg": { + "precision": 0.931049032633747, + "recall": 0.8757506709409886, + "f1-score": 0.9000814889173989, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9416053167583962, + "recall": 0.942626604850214, + "f1-score": 0.9406635117679449, + "support": 224320.0 + }, + "cross-entropy": 0.13436786219631347, + "roc-auc": 0.9804692077039372, + "score": 0.9804692077039372 + }, + "val": { + "0": { + "precision": 0.9353075170842825, + "recall": 0.9810644525416642, + "f1-score": 0.957639718958631, + "support": 16741.0 + }, + "1": { + "precision": 0.8770364623739333, + "recall": 0.6655872828966736, + "f1-score": 0.7568200836820084, + "support": 3397.0 + }, + "accuracy": 0.9278478498361307, + "macro avg": { + "precision": 0.9061719897291078, + "recall": 0.8233258677191688, + "f1-score": 0.8572299013203197, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9254780020951546, + "recall": 0.9278478498361307, + "f1-score": 0.9237642446794232, + "support": 20138.0 + }, + "cross-entropy": 0.1724175473381701, + "roc-auc": 0.9589599740471012, + "score": 0.9589599740471012 + }, + "test": { + "0": { + "precision": 0.931104503589818, + "recall": 0.9765726021145509, + "f1-score": 0.9532967032967034, + "support": 13147.0 + }, + "1": { + "precision": 0.8770949720670391, + "recall": 0.6982210927573063, + "f1-score": 0.7775026529890343, + "support": 3148.0 + }, + "accuracy": 0.9227984044185333, + "macro avg": { + "precision": 0.9040997378284286, + "recall": 0.8373968474359286, + "f1-score": 0.8653996781428688, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9206705051096887, + "recall": 0.9227984044185333, + "f1-score": 0.919335385692006, + "support": 16295.0 + }, + "cross-entropy": 0.17668487055650292, + "roc-auc": 0.9628908822909436, + "score": 0.9628908822909436 + } + }, + "time": "0:03:46.650441", + "diversity": { + "train": { + "mean_centered_corr": 0.0506281740963459, + "mean_pairwise_disagreement": 0.03327413710540242, + "member_std": 0.03079444356262684 + }, + "val": { + "mean_centered_corr": 0.056434109807014465, + "mean_pairwise_disagreement": 0.030276074921749604, + "member_std": 0.03229342773556709 + }, + "test": { + "mean_centered_corr": 0.06734751909971237, + "mean_pairwise_disagreement": 0.037917825574834946, + "member_std": 0.03452758118510246 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..a6950ee69 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/cf_fisd_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,639 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2420, + "metrics": { + "train": { + "0": { + "precision": 0.9470743368373016, + "recall": 0.9808101290865028, + "f1-score": 0.9636470639051465, + "support": 181971.0 + }, + "1": { + "precision": 0.9026403100342933, + "recall": 0.7644808614134927, + "f1-score": 0.8278357369336197, + "support": 42349.0 + }, + "accuracy": 0.9399696861626248, + "macro avg": { + "precision": 0.9248573234357975, + "recall": 0.8726454952499978, + "f1-score": 0.8957414004193831, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9386857107625842, + "recall": 0.9399696861626248, + "f1-score": 0.9380074691926055, + "support": 224320.0 + }, + "cross-entropy": 0.13890143673129085, + "roc-auc": 0.9784781707405066, + "score": 0.9784781707405066 + }, + "val": { + "0": { + "precision": 0.9363594890510949, + "recall": 0.9808255181888776, + "f1-score": 0.9580768445313184, + "support": 16741.0 + }, + "1": { + "precision": 0.8766333589546502, + "recall": 0.6714748307329997, + "f1-score": 0.7604600766794466, + "support": 3397.0 + }, + "accuracy": 0.9286423676631245, + "macro avg": { + "precision": 0.9064964240028726, + "recall": 0.8261501744609387, + "f1-score": 0.8592684606053824, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9262845231191442, + "recall": 0.9286423676631245, + "f1-score": 0.9247416493583713, + "support": 20138.0 + }, + "cross-entropy": 0.17179269209939912, + "roc-auc": 0.9594844321379927, + "score": 0.9594844321379927 + }, + "test": { + "0": { + "precision": 0.9321799811005307, + "recall": 0.9754316574123374, + "f1-score": 0.9533154921201308, + "support": 13147.0 + }, + "1": { + "precision": 0.8727344365642238, + "recall": 0.7036213468869124, + "f1-score": 0.7791065775589167, + "support": 3148.0 + }, + "accuracy": 0.9229211414544338, + "macro avg": { + "precision": 0.9024572088323772, + "recall": 0.8395265021496249, + "f1-score": 0.8662110348395238, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9206958096245998, + "recall": 0.9229211414544338, + "f1-score": 0.9196604038698268, + "support": 16295.0 + }, + "cross-entropy": 0.1758390088286691, + "roc-auc": 0.9634382071404677, + "score": 0.9634382071404677 + } + }, + "time": "0:03:46.650441", + "diversity": { + "train": { + "mean_centered_corr": 0.0506281740963459, + "mean_pairwise_disagreement": 0.03327413710540242, + "member_std": 0.03079444356262684 + }, + "val": { + "mean_centered_corr": 0.056434109807014465, + "mean_pairwise_disagreement": 0.030276074921749604, + "member_std": 0.03229342773556709 + }, + "test": { + "mean_centered_corr": 0.06734751909971237, + "mean_pairwise_disagreement": 0.037917825574834946, + "member_std": 0.03452758118510246 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 2, + 0, + 28, + 7, + 26, + 18, + 29, + 24, + 15, + 4, + 7, + 27, + 19, + 31, + 0, + 2, + 28, + 11 + ], + "head_selection_time": "0:00:17.095036" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f143c209a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-best-head-evaluation/0/report.json @@ -0,0 +1,272 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9545321160743397, + "recall": 0.9652801820070231, + "f1-score": 0.9598760625261277, + "support": 181971.0 + }, + "1": { + "precision": 0.8432258064516129, + "recall": 0.8024274481097546, + "f1-score": 0.8223208992244311, + "support": 42349.0 + }, + "accuracy": 0.934535485021398, + "macro avg": { + "precision": 0.8988789612629764, + "recall": 0.8838538150583889, + "f1-score": 0.8910984808752793, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9335187828619073, + "recall": 0.934535485021398, + "f1-score": 0.9339072518509157, + "support": 224320.0 + }, + "cross-entropy": 0.1498442336366005, + "roc-auc": 0.9747257011404723, + "score": 0.9747257011404723 + }, + "val": { + "0": { + "precision": 0.9430700273589848, + "recall": 0.9677438623738128, + "f1-score": 0.955247641509434, + "support": 16741.0 + }, + "1": { + "precision": 0.8175059141601893, + "recall": 0.7120989108036503, + "f1-score": 0.7611705475141598, + "support": 3397.0 + }, + "accuracy": 0.9246201211639686, + "macro avg": { + "precision": 0.880287970759587, + "recall": 0.8399213865887316, + "f1-score": 0.8582090945117968, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9218891110546691, + "recall": 0.9246201211639686, + "f1-score": 0.9225095399947877, + "support": 20138.0 + }, + "cross-entropy": 0.18072654373317865, + "roc-auc": 0.9561512908829329, + "score": 0.9561512908829329 + }, + "test": { + "0": { + "precision": 0.938474098263322, + "recall": 0.9618163839659238, + "f1-score": 0.9500018782164457, + "support": 13147.0 + }, + "1": { + "precision": 0.8220489188231124, + "recall": 0.7366581956797967, + "f1-score": 0.7770145753057464, + "support": 3148.0 + }, + "accuracy": 0.9183185026081621, + "macro avg": { + "precision": 0.8802615085432172, + "recall": 0.8492372898228603, + "f1-score": 0.8635082267610961, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9159821396945721, + "recall": 0.9183185026081621, + "f1-score": 0.9165827907931329, + "support": 16295.0 + }, + "cross-entropy": 0.18568286621420937, + "roc-auc": 0.9606910602029306, + "score": 0.9606910602029306 + } + }, + "time": "0:05:02.257608", + "diversity": { + "train": { + "mean_centered_corr": 0.049995824694633484, + "mean_pairwise_disagreement": 0.03397909919872533, + "member_std": 0.031108465045690536 + }, + "val": { + "mean_centered_corr": 0.05969587340950966, + "mean_pairwise_disagreement": 0.03266793800197989, + "member_std": 0.03460051864385605 + }, + "test": { + "mean_centered_corr": 0.07376471161842346, + "mean_pairwise_disagreement": 0.04182474833958566, + "member_std": 0.03703681379556656 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 17 + ], + "head_selection_time": "0:00:00.953794" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0.toml new file mode 100644 index 000000000..016b99992 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0/report.json new file mode 100644 index 000000000..b0473609b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0/report.json @@ -0,0 +1,268 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9496102021429594, + "recall": 0.9833215182638992, + "f1-score": 0.9661718889206862, + "support": 181971.0 + }, + "1": { + "precision": 0.9154336983476832, + "recall": 0.7757916361661432, + "f1-score": 0.8398476443671873, + "support": 42349.0 + }, + "accuracy": 0.9441422967189729, + "macro avg": { + "precision": 0.9325219502453213, + "recall": 0.8795565772150212, + "f1-score": 0.9030097666439367, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9431580767897758, + "recall": 0.9441422967189729, + "f1-score": 0.9423233447311528, + "support": 224320.0 + }, + "cross-entropy": 0.13329761494760606, + "roc-auc": 0.9810524262916843, + "score": 0.9810524262916843 + }, + "val": { + "0": { + "precision": 0.936472602739726, + "recall": 0.9800489815423212, + "f1-score": 0.9577653892180613, + "support": 16741.0 + }, + "1": { + "precision": 0.8724216959511077, + "recall": 0.6723579629084486, + "f1-score": 0.7594347464671655, + "support": 3397.0 + }, + "accuracy": 0.9281457940212533, + "macro avg": { + "precision": 0.9044471493454169, + "recall": 0.8262034722253849, + "f1-score": 0.8586000678426133, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9256681072406231, + "recall": 0.9281457940212533, + "f1-score": 0.9243097732966793, + "support": 20138.0 + }, + "cross-entropy": 0.1734408298186842, + "roc-auc": 0.9583120395781356, + "score": 0.9583120395781356 + }, + "test": { + "0": { + "precision": 0.9324796274738067, + "recall": 0.9748231535711569, + "f1-score": 0.9531813617939088, + "support": 13147.0 + }, + "1": { + "precision": 0.8702469619756958, + "recall": 0.7052096569250318, + "f1-score": 0.779084049833304, + "support": 3148.0 + }, + "accuracy": 0.922737035900583, + "macro avg": { + "precision": 0.9013632947247513, + "recall": 0.8400164052480943, + "f1-score": 0.8661327058136064, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9204570174101029, + "recall": 0.922737035900583, + "f1-score": 0.9195478338373586, + "support": 16295.0 + }, + "cross-entropy": 0.17694458755574416, + "roc-auc": 0.9627483366901237, + "score": 0.9627483366901237 + } + }, + "time": "0:05:02.257608", + "diversity": { + "train": { + "mean_centered_corr": 0.049995824694633484, + "mean_pairwise_disagreement": 0.03397909919872533, + "member_std": 0.031108465045690536 + }, + "val": { + "mean_centered_corr": 0.05969587340950966, + "mean_pairwise_disagreement": 0.03266793800197989, + "member_std": 0.03460051864385605 + }, + "test": { + "mean_centered_corr": 0.07376471161842346, + "mean_pairwise_disagreement": 0.04182474833958566, + "member_std": 0.03703681379556656 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..55f982fc5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,290 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9495084683651196, + "recall": 0.9824752295695468, + "f1-score": 0.9657105817533626, + "support": 181971.0 + }, + "1": { + "precision": 0.9114928811301379, + "recall": 0.7755082764646155, + "f1-score": 0.8380199030364889, + "support": 42349.0 + }, + "accuracy": 0.9434022824536377, + "macro avg": { + "precision": 0.9305006747476288, + "recall": 0.8789917530170812, + "f1-score": 0.9018652423949257, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9423315688295711, + "recall": 0.9434022824536377, + "f1-score": 0.9416040707290184, + "support": 224320.0 + }, + "cross-entropy": 0.13410595787789584, + "roc-auc": 0.9804656245919036, + "score": 0.9804656245919036 + }, + "val": { + "0": { + "precision": 0.9362795477903392, + "recall": 0.9795113792485515, + "f1-score": 0.957407677711283, + "support": 16741.0 + }, + "1": { + "precision": 0.8692835365853658, + "recall": 0.6714748307329997, + "f1-score": 0.7576814482644079, + "support": 3397.0 + }, + "accuracy": 0.927549905651008, + "macro avg": { + "precision": 0.9027815421878524, + "recall": 0.8254931049907757, + "f1-score": 0.8575445629878454, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9249782542128591, + "recall": 0.927549905651008, + "f1-score": 0.9237166457601939, + "support": 20138.0 + }, + "cross-entropy": 0.17278545942960094, + "roc-auc": 0.958991133281215, + "score": 0.958991133281215 + }, + "test": { + "0": { + "precision": 0.9319371727748691, + "recall": 0.9748231535711569, + "f1-score": 0.9528978772445073, + "support": 13147.0 + }, + "1": { + "precision": 0.8698387731026347, + "recall": 0.7026683608640406, + "f1-score": 0.7773677736777368, + "support": 3148.0 + }, + "accuracy": 0.9222460877569807, + "macro avg": { + "precision": 0.9008879729387519, + "recall": 0.8387457572175987, + "f1-score": 0.865132825461122, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9199405012702239, + "recall": 0.9222460877569807, + "f1-score": 0.9189875510077357, + "support": 16295.0 + }, + "cross-entropy": 0.1769733127255982, + "roc-auc": 0.9627938464179217, + "score": 0.9627938464179217 + } + }, + "time": "0:05:02.257608", + "diversity": { + "train": { + "mean_centered_corr": 0.049995824694633484, + "mean_pairwise_disagreement": 0.03397909919872533, + "member_std": 0.031108465045690536 + }, + "val": { + "mean_centered_corr": 0.05969587340950966, + "mean_pairwise_disagreement": 0.03266793800197989, + "member_std": 0.03460051864385605 + }, + "test": { + "mean_centered_corr": 0.07376471161842346, + "mean_pairwise_disagreement": 0.04182474833958566, + "member_std": 0.03703681379556656 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 17, + 23, + 5, + 26, + 21, + 24, + 30, + 9, + 28, + 15, + 22, + 12, + 20, + 29, + 23, + 26, + 5, + 21, + 30 + ], + "head_selection_time": "0:00:17.134347" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..490610baf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-best-head-evaluation/0/report.json @@ -0,0 +1,298 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9518098162802934, + "recall": 0.9742541393958378, + "f1-score": 0.9629012065730121, + "support": 181971.0 + }, + "1": { + "precision": 0.8768984182037942, + "recall": 0.7880469432572198, + "f1-score": 0.8301018568035121, + "support": 42349.0 + }, + "accuracy": 0.9391003922967189, + "macro avg": { + "precision": 0.9143541172420437, + "recall": 0.8811505413265288, + "f1-score": 0.8965015316882621, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9376674179335491, + "recall": 0.9391003922967189, + "f1-score": 0.9378302380308022, + "support": 224320.0 + }, + "cross-entropy": 0.1417006278725686, + "roc-auc": 0.9768342693042886, + "score": 0.9768342693042886 + }, + "val": { + "0": { + "precision": 0.9406342396660676, + "recall": 0.9691774684905322, + "f1-score": 0.9546925566343042, + "support": 16741.0 + }, + "1": { + "precision": 0.8213914849428868, + "recall": 0.6985575507801001, + "f1-score": 0.755011135857461, + "support": 3397.0 + }, + "accuracy": 0.9235276591518522, + "macro avg": { + "precision": 0.8810128623044773, + "recall": 0.8338675096353161, + "f1-score": 0.8548518462458826, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9205196484556869, + "recall": 0.9235276591518522, + "f1-score": 0.9210090832814918, + "support": 20138.0 + }, + "cross-entropy": 0.18275894278340477, + "roc-auc": 0.9560981250001208, + "score": 0.9560981250001208 + }, + "test": { + "0": { + "precision": 0.9386612939332886, + "recall": 0.9567962272761846, + "f1-score": 0.9476420069308422, + "support": 13147.0 + }, + "1": { + "precision": 0.8037318590186593, + "recall": 0.738881829733164, + "f1-score": 0.7699437272426349, + "support": 3148.0 + }, + "accuracy": 0.9146977600490949, + "macro avg": { + "precision": 0.8711965764759739, + "recall": 0.8478390285046743, + "f1-score": 0.8587928670867386, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9125945335091553, + "recall": 0.9146977600490949, + "f1-score": 0.9133128148806136, + "support": 16295.0 + }, + "cross-entropy": 0.1874554754465997, + "roc-auc": 0.9601610718172741, + "score": 0.9601610718172741 + } + }, + "time": "0:03:06.973678", + "diversity": { + "train": { + "mean_centered_corr": 0.08821944147348404, + "mean_pairwise_disagreement": 0.03961999131425153, + "member_std": 0.03912428021430969 + }, + "val": { + "mean_centered_corr": 0.09983313083648682, + "mean_pairwise_disagreement": 0.03728577252442021, + "member_std": 0.04294782876968384 + }, + "test": { + "mean_centered_corr": 0.11663155257701874, + "mean_pairwise_disagreement": 0.04569306832691603, + "member_std": 0.04552939534187317 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:01.054713" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0.toml new file mode 100644 index 000000000..fe5f765a4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..5f7ccee67 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-evaluation/0/report.json @@ -0,0 +1,294 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.946151494708437, + "recall": 0.9865253254639476, + "f1-score": 0.9659167032097862, + "support": 181971.0 + }, + "1": { + "precision": 0.9291001619245894, + "recall": 0.7587428274575551, + "f1-score": 0.8353242431726308, + "support": 42349.0 + }, + "accuracy": 0.9435226462196862, + "macro avg": { + "precision": 0.9376258283165132, + "recall": 0.8726340764607513, + "f1-score": 0.9006204731912084, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9429324019299814, + "recall": 0.9435226462196862, + "f1-score": 0.94126236971249, + "support": 224320.0 + }, + "cross-entropy": 0.13654273925686128, + "roc-auc": 0.980952103177431, + "score": 0.980952103177431 + }, + "val": { + "0": { + "precision": 0.9326650809831238, + "recall": 0.9837524640105131, + "f1-score": 0.9575278351114858, + "support": 16741.0 + }, + "1": { + "precision": 0.8903225806451613, + "recall": 0.6499852811304092, + "f1-score": 0.7514037774374681, + "support": 3397.0 + }, + "accuracy": 0.9274505909226338, + "macro avg": { + "precision": 0.9114938308141426, + "recall": 0.8168688725704611, + "f1-score": 0.854465806274477, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9255224911704286, + "recall": 0.9274505909226338, + "f1-score": 0.9227575786848973, + "support": 20138.0 + }, + "cross-entropy": 0.17552008158175839, + "roc-auc": 0.9580942942079151, + "score": 0.9580942942079151 + }, + "test": { + "0": { + "precision": 0.9282410752998989, + "recall": 0.9770289799954363, + "f1-score": 0.9520103761348897, + "support": 13147.0 + }, + "1": { + "precision": 0.877085877085877, + "recall": 0.684561626429479, + "f1-score": 0.7689562890276539, + "support": 3148.0 + }, + "accuracy": 0.9205277692543725, + "macro avg": { + "precision": 0.902663476192888, + "recall": 0.8307953032124576, + "f1-score": 0.8604833325812717, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9183585000327775, + "recall": 0.9205277692543725, + "f1-score": 0.9166465058548297, + "support": 16295.0 + }, + "cross-entropy": 0.1804777281050399, + "roc-auc": 0.9616652728230258, + "score": 0.9616652728230258 + } + }, + "time": "0:03:06.973678", + "diversity": { + "train": { + "mean_centered_corr": 0.08821944147348404, + "mean_pairwise_disagreement": 0.03961999131425153, + "member_std": 0.03912428021430969 + }, + "val": { + "mean_centered_corr": 0.09983313083648682, + "mean_pairwise_disagreement": 0.03728577252442021, + "member_std": 0.04294782876968384 + }, + "test": { + "mean_centered_corr": 0.11663155257701874, + "mean_pairwise_disagreement": 0.04569306832691603, + "member_std": 0.04552939534187317 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..334ff2f4b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/mfb_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,307 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.946386196568126, + "recall": 0.9862395656450753, + "f1-score": 0.9659019657970156, + "support": 181971.0 + }, + "1": { + "precision": 0.9278094908608661, + "recall": 0.7599234928805875, + "f1-score": 0.8355163237489452, + "support": 42349.0 + }, + "accuracy": 0.9435137303851641, + "macro avg": { + "precision": 0.937097843714496, + "recall": 0.8730815292628313, + "f1-score": 0.9007091447729805, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9428791311704943, + "recall": 0.9435137303851641, + "f1-score": 0.9412866771241655, + "support": 224320.0 + }, + "cross-entropy": 0.13649569775840315, + "roc-auc": 0.9805235149654821, + "score": 0.9805235149654821 + }, + "val": { + "0": { + "precision": 0.9322954995754317, + "recall": 0.9837524640105131, + "f1-score": 0.9573330233098878, + "support": 16741.0 + }, + "1": { + "precision": 0.8900121310149616, + "recall": 0.6479246393876951, + "f1-score": 0.7499148211243611, + "support": 3397.0 + }, + "accuracy": 0.927102989373324, + "macro avg": { + "precision": 0.9111538152951966, + "recall": 0.815838551699104, + "f1-score": 0.8536239222171245, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9251628844696655, + "recall": 0.927102989373324, + "f1-score": 0.9223444627366316, + "support": 20138.0 + }, + "cross-entropy": 0.17444519969608893, + "roc-auc": 0.9587921238951639, + "score": 0.9587921238951639 + }, + "test": { + "0": { + "precision": 0.9292688218702538, + "recall": 0.9773332319160265, + "f1-score": 0.9526951879587752, + "support": 13147.0 + }, + "1": { + "precision": 0.8792544570502431, + "recall": 0.6893265565438373, + "f1-score": 0.7727920227920227, + "support": 3148.0 + }, + "accuracy": 0.921693771095428, + "macro avg": { + "precision": 0.9042616394602485, + "recall": 0.8333298942299319, + "f1-score": 0.862743605375399, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9196066420326721, + "recall": 0.921693771095428, + "f1-score": 0.9179400382843391, + "support": 16295.0 + }, + "cross-entropy": 0.1794710390286192, + "roc-auc": 0.9622146031450255, + "score": 0.9622146031450255 + } + }, + "time": "0:03:06.973678", + "diversity": { + "train": { + "mean_centered_corr": 0.08821944147348404, + "mean_pairwise_disagreement": 0.03961999131425153, + "member_std": 0.03912428021430969 + }, + "val": { + "mean_centered_corr": 0.09983313083648682, + "mean_pairwise_disagreement": 0.03728577252442021, + "member_std": 0.04294782876968384 + }, + "test": { + "mean_centered_corr": 0.11663155257701874, + "mean_pairwise_disagreement": 0.04569306832691603, + "member_std": 0.04552939534187317 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 7, + 23, + 22, + 15, + 5, + 2, + 30, + 7, + 28 + ], + "head_selection_time": "0:00:10.953573" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6416ea4eb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-best-head-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9470929244882358, + "recall": 0.9803869847393266, + "f1-score": 0.9634524044186542, + "support": 181971.0 + }, + "1": { + "precision": 0.9007287494437027, + "recall": 0.7646697678811778, + "f1-score": 0.8271414158184442, + "support": 42349.0 + }, + "accuracy": 0.9396620898716119, + "macro avg": { + "precision": 0.9239108369659692, + "recall": 0.8725283763102523, + "f1-score": 0.8952969101185493, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9383399089347366, + "recall": 0.9396620898716119, + "f1-score": 0.9377184794176276, + "support": 224320.0 + }, + "cross-entropy": 0.1374541129990154, + "roc-auc": 0.9783636099059336, + "score": 0.9783636099059336 + }, + "val": { + "0": { + "precision": 0.9391985260248733, + "recall": 0.9743742906636401, + "f1-score": 0.9564631035797003, + "support": 16741.0 + }, + "1": { + "precision": 0.8451263537906137, + "recall": 0.6891374742419782, + "f1-score": 0.7592022052862007, + "support": 3397.0 + }, + "accuracy": 0.9262588141821432, + "macro avg": { + "precision": 0.8921624399077435, + "recall": 0.8317558824528092, + "f1-score": 0.8578326544329505, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9233298613570919, + "recall": 0.9262588141821432, + "f1-score": 0.9231879386426154, + "support": 20138.0 + }, + "cross-entropy": 0.18034048370527975, + "roc-auc": 0.9557610707114682, + "score": 0.9557610707114682 + }, + "test": { + "0": { + "precision": 0.9347106830380119, + "recall": 0.9669886666159581, + "f1-score": 0.9505757439808584, + "support": 13147.0 + }, + "1": { + "precision": 0.8389012620638456, + "recall": 0.7179161372299873, + "f1-score": 0.7737076343717905, + "support": 3148.0 + }, + "accuracy": 0.9188708192697146, + "macro avg": { + "precision": 0.8868059725509287, + "recall": 0.8424524019229727, + "f1-score": 0.8621416891763245, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.916201443564144, + "recall": 0.9188708192697146, + "f1-score": 0.9164069309063357, + "support": 16295.0 + }, + "cross-entropy": 0.18574135097736744, + "roc-auc": 0.9604946447119459, + "score": 0.9604946447119459 + } + }, + "time": "0:07:11.953824", + "diversity": { + "train": { + "mean_centered_corr": 0.05171258747577667, + "mean_pairwise_disagreement": 0.03286417948437716, + "member_std": 0.03040449135005474 + }, + "val": { + "mean_centered_corr": 0.061806585639715195, + "mean_pairwise_disagreement": 0.03182976975001522, + "member_std": 0.0331539586186409 + }, + "test": { + "mean_centered_corr": 0.0732795000076294, + "mean_pairwise_disagreement": 0.03959877856852983, + "member_std": 0.03512473404407501 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.945023" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0.toml new file mode 100644 index 000000000..d6037d45d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0/report.json new file mode 100644 index 000000000..f1b6781b1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-evaluation/0/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9493414440624174, + "recall": 0.9862725379318683, + "f1-score": 0.9674546723770353, + "support": 181971.0 + }, + "1": { + "precision": 0.9291749362064077, + "recall": 0.7738553448723701, + "f1-score": 0.8444324198971901, + "support": 42349.0 + }, + "accuracy": 0.9461706490727532, + "macro avg": { + "precision": 0.9392581901344126, + "recall": 0.8800639414021192, + "f1-score": 0.9059435461371127, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9455342425592338, + "recall": 0.9461706490727532, + "f1-score": 0.9442295057834682, + "support": 224320.0 + }, + "cross-entropy": 0.1285332583359511, + "roc-auc": 0.9826472071386247, + "score": 0.9826472071386247 + }, + "val": { + "0": { + "precision": 0.9352083807788658, + "recall": 0.9811839197180575, + "f1-score": 0.9576446582130884, + "support": 16741.0 + }, + "1": { + "precision": 0.8776223776223776, + "recall": 0.6649985281130409, + "f1-score": 0.7566571763523698, + "support": 3397.0 + }, + "accuracy": 0.9278478498361307, + "macro avg": { + "precision": 0.9064153792006218, + "recall": 0.8230912239155492, + "f1-score": 0.8571509172827292, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9254944244414645, + "recall": 0.9278478498361307, + "f1-score": 0.9237408705538938, + "support": 20138.0 + }, + "cross-entropy": 0.173981665844509, + "roc-auc": 0.9583231528038466, + "score": 0.9583231528038466 + }, + "test": { + "0": { + "precision": 0.9317207815791385, + "recall": 0.9756598463527801, + "f1-score": 0.9531842163929554, + "support": 13147.0 + }, + "1": { + "precision": 0.8734177215189873, + "recall": 0.7013977128335451, + "f1-score": 0.7780126849894292, + "support": 3148.0 + }, + "accuracy": 0.9226756673826327, + "macro avg": { + "precision": 0.9025692515490629, + "recall": 0.8385287795931626, + "f1-score": 0.8655984506911922, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9204573245021606, + "recall": 0.9226756673826327, + "f1-score": 0.9193431620291443, + "support": 16295.0 + }, + "cross-entropy": 0.17786928687791626, + "roc-auc": 0.96242593886798, + "score": 0.96242593886798 + } + }, + "time": "0:07:11.953824", + "diversity": { + "train": { + "mean_centered_corr": 0.05171258747577667, + "mean_pairwise_disagreement": 0.03286417948437716, + "member_std": 0.03040449135005474 + }, + "val": { + "mean_centered_corr": 0.061806585639715195, + "mean_pairwise_disagreement": 0.03182976975001522, + "member_std": 0.0331539586186409 + }, + "test": { + "mean_centered_corr": 0.0732795000076294, + "mean_pairwise_disagreement": 0.03959877856852983, + "member_std": 0.03512473404407501 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..427f43a81 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/homesite-insurance/rla_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,285 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.946255084192114, + "recall": 0.9869869374790489, + "f1-score": 0.9661919171541927, + "support": 181971.0 + }, + "1": { + "precision": 0.931394136052845, + "recall": 0.7591206403929255, + "f1-score": 0.836479542054251, + "support": 42349.0 + }, + "accuracy": 0.9439684379457918, + "macro avg": { + "precision": 0.9388246101224795, + "recall": 0.8730537889359872, + "f1-score": 0.9013357296042219, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9434495104904829, + "recall": 0.9439684379457918, + "f1-score": 0.9417037334295696, + "support": 224320.0 + }, + "cross-entropy": 0.1317045399668242, + "roc-auc": 0.9813354497094697, + "score": 0.9813354497094697 + }, + "val": { + "0": { + "precision": 0.9333144315282109, + "recall": 0.9831551281285467, + "f1-score": 0.9575866883872469, + "support": 16741.0 + }, + "1": { + "precision": 0.8873351977626848, + "recall": 0.6538121872240212, + "f1-score": 0.7528813559322034, + "support": 3397.0 + }, + "accuracy": 0.9275995630151952, + "macro avg": { + "precision": 0.9103248146454479, + "recall": 0.818483657676284, + "f1-score": 0.8552340221597252, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.925558375460007, + "recall": 0.9275995630151952, + "f1-score": 0.9230557511367858, + "support": 20138.0 + }, + "cross-entropy": 0.1731441036701711, + "roc-auc": 0.959029757015826, + "score": 0.959029757015826 + }, + "test": { + "0": { + "precision": 0.9292688218702538, + "recall": 0.9773332319160265, + "f1-score": 0.9526951879587752, + "support": 13147.0 + }, + "1": { + "precision": 0.8792544570502431, + "recall": 0.6893265565438373, + "f1-score": 0.7727920227920227, + "support": 3148.0 + }, + "accuracy": 0.921693771095428, + "macro avg": { + "precision": 0.9042616394602485, + "recall": 0.8333298942299319, + "f1-score": 0.862743605375399, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9196066420326721, + "recall": 0.921693771095428, + "f1-score": 0.9179400382843391, + "support": 16295.0 + }, + "cross-entropy": 0.17812450933232857, + "roc-auc": 0.9624579176971494, + "score": 0.9624579176971494 + } + }, + "time": "0:07:11.953824", + "diversity": { + "train": { + "mean_centered_corr": 0.05171258747577667, + "mean_pairwise_disagreement": 0.03286417948437716, + "member_std": 0.03040449135005474 + }, + "val": { + "mean_centered_corr": 0.061806585639715195, + "mean_pairwise_disagreement": 0.03182976975001522, + "member_std": 0.0331539586186409 + }, + "test": { + "mean_centered_corr": 0.0732795000076294, + "mean_pairwise_disagreement": 0.03959877856852983, + "member_std": 0.03512473404407501 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 22, + 6, + 20, + 26, + 21, + 24, + 7, + 22, + 13, + 5 + ], + "head_selection_time": "0:00:11.473881" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8d841fedb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-best-head-evaluation/0/report.json @@ -0,0 +1,702 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.238513825213292, + "mae": 0.12740404903888702, + "r2": 0.6365530490875244, + "score": -0.238513825213292 + }, + "val": { + "rmse": 0.2376718937442043, + "mae": 0.13975045084953308, + "r2": 0.6226825714111328, + "score": -0.2376718937442043 + }, + "test": { + "rmse": 0.2539783962159738, + "mae": 0.17701080441474915, + "r2": 0.5592085719108582, + "score": -0.2539783962159738 + } + }, + "time": "0:03:30.033690", + "diversity": { + "train": { + "mean_centered_corr": -0.018900908529758453, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04601670801639557 + }, + "val": { + "mean_centered_corr": -0.01799248345196247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05731586739420891 + }, + "test": { + "mean_centered_corr": -0.019703488796949387, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07458258420228958 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015071" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..aaa491b98 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0.toml @@ -0,0 +1,75 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..ae0cfb53b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2331321585964892, + "mae": 0.12105797231197357, + "r2": 0.6527692079544067, + "score": -0.2331321585964892 + }, + "val": { + "rmse": 0.2348792849855834, + "mae": 0.12763135135173798, + "r2": 0.6314972639083862, + "score": -0.2348792849855834 + }, + "test": { + "rmse": 0.2326272249658597, + "mae": 0.1447165459394455, + "r2": 0.6302052736282349, + "score": -0.2326272249658597 + } + }, + "time": "0:03:30.033690", + "diversity": { + "train": { + "mean_centered_corr": -0.018900908529758453, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04601670801639557 + }, + "val": { + "mean_centered_corr": -0.01799248345196247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05731586739420891 + }, + "test": { + "mean_centered_corr": -0.019703488796949387, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07458258420228958 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0f6ab25ae --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,710 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23288428364808966, + "mae": 0.12141954898834229, + "r2": 0.6535071730613708, + "score": -0.23288428364808966 + }, + "val": { + "rmse": 0.23335850745168515, + "mae": 0.1262979656457901, + "r2": 0.6362537145614624, + "score": -0.23335850745168515 + }, + "test": { + "rmse": 0.23263890687901315, + "mae": 0.14496733248233795, + "r2": 0.6301681995391846, + "score": -0.23263890687901315 + } + }, + "time": "0:03:30.033690", + "diversity": { + "train": { + "mean_centered_corr": -0.018900908529758453, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04601670801639557 + }, + "val": { + "mean_centered_corr": -0.01799248345196247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05731586739420891 + }, + "test": { + "mean_centered_corr": -0.019703488796949387, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07458258420228958 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 10, + 19, + 8, + 12, + 28, + 21, + 24, + 20 + ], + "head_selection_time": "0:00:00.144155" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-best-head-evaluation/0/report.json new file mode 100644 index 000000000..546302ba0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-best-head-evaluation/0/report.json @@ -0,0 +1,190 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22658863410177663, + "mae": 0.12739618122577667, + "r2": 0.6719876527786255, + "score": -0.22658863410177663 + }, + "val": { + "rmse": 0.23955454428788905, + "mae": 0.13915589451789856, + "r2": 0.6166812181472778, + "score": -0.23955454428788905 + }, + "test": { + "rmse": 0.26588264760298186, + "mae": 0.1863766759634018, + "r2": 0.5169194340705872, + "score": -0.26588264760298186 + } + }, + "time": "0:00:48.800887", + "diversity": { + "train": { + "mean_centered_corr": -0.018623020499944687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05047355964779854 + }, + "val": { + "mean_centered_corr": -0.01870153285562992, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06864207983016968 + }, + "test": { + "mean_centered_corr": -0.019026711583137512, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09529124945402145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015244" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0.toml new file mode 100644 index 000000000..a68e82651 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0.toml @@ -0,0 +1,34 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0/report.json new file mode 100644 index 000000000..c4a6eed6f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0/report.json @@ -0,0 +1,186 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22224988999004666, + "mae": 0.11853151023387909, + "r2": 0.6844290494918823, + "score": -0.22224988999004666 + }, + "val": { + "rmse": 0.23452539387100468, + "mae": 0.12745676934719086, + "r2": 0.6326068639755249, + "score": -0.23452539387100468 + }, + "test": { + "rmse": 0.23846659721529556, + "mae": 0.15488465130329132, + "r2": 0.6114072799682617, + "score": -0.23846659721529556 + } + }, + "time": "0:00:48.800887", + "diversity": { + "train": { + "mean_centered_corr": -0.018623020499944687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05047355964779854 + }, + "val": { + "mean_centered_corr": -0.01870153285562992, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06864207983016968 + }, + "test": { + "mean_centered_corr": -0.019026711583137512, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09529124945402145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d2edb64f2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/baseline_plr-greedy-heads-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22302489965783162, + "mae": 0.11715596914291382, + "r2": 0.6822243928909302, + "score": -0.22302489965783162 + }, + "val": { + "rmse": 0.23257941028161666, + "mae": 0.12573255598545074, + "r2": 0.6386785507202148, + "score": -0.23257941028161666 + }, + "test": { + "rmse": 0.23549883338311248, + "mae": 0.14942243695259094, + "r2": 0.6210193037986755, + "score": -0.23549883338311248 + } + }, + "time": "0:00:48.800887", + "diversity": { + "train": { + "mean_centered_corr": -0.018623020499944687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05047355964779854 + }, + "val": { + "mean_centered_corr": -0.01870153285562992, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06864207983016968 + }, + "test": { + "mean_centered_corr": -0.019026711583137512, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09529124945402145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 19, + 21, + 12, + 16, + 18, + 8, + 22, + 19, + 29 + ], + "head_selection_time": "0:00:00.158388" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bfcad143f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-best-head-evaluation/0/report.json @@ -0,0 +1,643 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24287177140617242, + "mae": 0.12792739272117615, + "r2": 0.6231504678726196, + "score": -0.24287177140617242 + }, + "val": { + "rmse": 0.2388197411165918, + "mae": 0.13359415531158447, + "r2": 0.61902916431427, + "score": -0.2388197411165918 + }, + "test": { + "rmse": 0.2462523116085554, + "mae": 0.16663780808448792, + "r2": 0.5856186151504517, + "score": -0.2462523116085554 + } + }, + "time": "0:01:03.015496", + "diversity": { + "train": { + "mean_centered_corr": -0.018840299919247627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04810040444135666 + }, + "val": { + "mean_centered_corr": -0.020352300256490707, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061659909784793854 + }, + "test": { + "mean_centered_corr": -0.019725581631064415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08170976489782333 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015354" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0.toml new file mode 100644 index 000000000..523756861 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0.toml @@ -0,0 +1,45 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0/report.json new file mode 100644 index 000000000..747fa815c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0/report.json @@ -0,0 +1,639 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23279361488245573, + "mae": 0.12178773432970047, + "r2": 0.6537768840789795, + "score": -0.23279361488245573 + }, + "val": { + "rmse": 0.23523413233394516, + "mae": 0.12663741409778595, + "r2": 0.6303830146789551, + "score": -0.23523413233394516 + }, + "test": { + "rmse": 0.235149605552611, + "mae": 0.14915932714939117, + "r2": 0.6221424341201782, + "score": -0.235149605552611 + } + }, + "time": "0:01:03.015496", + "diversity": { + "train": { + "mean_centered_corr": -0.018840299919247627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04810040444135666 + }, + "val": { + "mean_centered_corr": -0.020352300256490707, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061659909784793854 + }, + "test": { + "mean_centered_corr": -0.019725581631064415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08170976489782333 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d85be1baa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/cf_fisd_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,649 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23477394006764257, + "mae": 0.1248951405286789, + "r2": 0.6478613615036011, + "score": -0.23477394006764257 + }, + "val": { + "rmse": 0.23327114512588593, + "mae": 0.12512965500354767, + "r2": 0.6365260481834412, + "score": -0.23327114512588593 + }, + "test": { + "rmse": 0.23367141178994547, + "mae": 0.14663751423358917, + "r2": 0.6268780827522278, + "score": -0.23367141178994547 + } + }, + "time": "0:01:03.015496", + "diversity": { + "train": { + "mean_centered_corr": -0.018840299919247627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04810040444135666 + }, + "val": { + "mean_centered_corr": -0.020352300256490707, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061659909784793854 + }, + "test": { + "mean_centered_corr": -0.019725581631064415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08170976489782333 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 21, + 25, + 19, + 0, + 28 + ], + "head_selection_time": "0:00:00.116743" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0eb3e3cee --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-best-head-evaluation/0/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2276013825171354, + "mae": 0.12862664461135864, + "r2": 0.6690490245819092, + "score": -0.2276013825171354 + }, + "val": { + "rmse": 0.2398028724080995, + "mae": 0.1366567611694336, + "r2": 0.6158860921859741, + "score": -0.2398028724080995 + }, + "test": { + "rmse": 0.23711371881317633, + "mae": 0.14537189900875092, + "r2": 0.6158038973808289, + "score": -0.23711371881317633 + } + }, + "time": "0:01:29.863372", + "diversity": { + "train": { + "mean_centered_corr": -0.017507946118712425, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0488579161465168 + }, + "val": { + "mean_centered_corr": -0.017691537737846375, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0768243595957756 + }, + "test": { + "mean_centered_corr": -0.014683594927191734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10134773701429367 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015232" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0.toml new file mode 100644 index 000000000..f5a2c1fd1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0/report.json new file mode 100644 index 000000000..0401b06c5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/0/report.json @@ -0,0 +1,196 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21979515700408836, + "mae": 0.12170511484146118, + "r2": 0.6913614869117737, + "score": -0.21979515700408836 + }, + "val": { + "rmse": 0.23533073111501152, + "mae": 0.12932802736759186, + "r2": 0.6300793886184692, + "score": -0.23533073111501152 + }, + "test": { + "rmse": 0.24162357572341994, + "mae": 0.15881586074829102, + "r2": 0.6010502576828003, + "score": -0.24162357572341994 + } + }, + "time": "0:01:29.863372", + "diversity": { + "train": { + "mean_centered_corr": -0.017507946118712425, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0488579161465168 + }, + "val": { + "mean_centered_corr": -0.017691537737846375, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0768243595957756 + }, + "test": { + "mean_centered_corr": -0.014683594927191734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10134773701429367 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9b882385e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22114994595231807, + "mae": 0.12296636402606964, + "r2": 0.6875449419021606, + "score": -0.22114994595231807 + }, + "val": { + "rmse": 0.23380275644783471, + "mae": 0.12705549597740173, + "r2": 0.6348674893379211, + "score": -0.23380275644783471 + }, + "test": { + "rmse": 0.24257603366460162, + "mae": 0.1593162715435028, + "r2": 0.5978988409042358, + "score": -0.24257603366460162 + } + }, + "time": "0:01:29.863372", + "diversity": { + "train": { + "mean_centered_corr": -0.017507946118712425, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0488579161465168 + }, + "val": { + "mean_centered_corr": -0.017691537737846375, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0768243595957756 + }, + "test": { + "mean_centered_corr": -0.014683594927191734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10134773701429367 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 25, + 0, + 16, + 30, + 22, + 25, + 31, + 16, + 22 + ], + "head_selection_time": "0:00:00.158598" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9ad34d612 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-best-head-evaluation/0/report.json @@ -0,0 +1,226 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.2105902710250961, + "mae": 0.11857529729604721, + "r2": 0.716671347618103, + "score": -0.2105902710250961 + }, + "val": { + "rmse": 0.24356476580119216, + "mae": 0.1343512386083603, + "r2": 0.6037400364875793, + "score": -0.24356476580119216 + }, + "test": { + "rmse": 0.2459548830871388, + "mae": 0.14966677129268646, + "r2": 0.5866190195083618, + "score": -0.2459548830871388 + } + }, + "time": "0:01:08.539078", + "diversity": { + "train": { + "mean_centered_corr": -0.016982607543468475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05261818319559097 + }, + "val": { + "mean_centered_corr": -0.017025893554091454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08680207282304764 + }, + "test": { + "mean_centered_corr": -0.01696028746664524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11552654951810837 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.015353" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0.toml new file mode 100644 index 000000000..da979e186 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0/report.json new file mode 100644 index 000000000..e931f6cc5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-evaluation/0/report.json @@ -0,0 +1,222 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20459342170445438, + "mae": 0.11115512996912003, + "r2": 0.732577919960022, + "score": -0.20459342170445438 + }, + "val": { + "rmse": 0.23520949733542967, + "mae": 0.1269780397415161, + "r2": 0.6304603815078735, + "score": -0.23520949733542967 + }, + "test": { + "rmse": 0.23880983570262945, + "mae": 0.15338461101055145, + "r2": 0.6102877855300903, + "score": -0.23880983570262945 + } + }, + "time": "0:01:08.539078", + "diversity": { + "train": { + "mean_centered_corr": -0.016982607543468475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05261818319559097 + }, + "val": { + "mean_centered_corr": -0.017025893554091454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08680207282304764 + }, + "test": { + "mean_centered_corr": -0.01696028746664524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11552654951810837 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..63ff10428 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/mfb_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,234 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20332438513408257, + "mae": 0.11058838665485382, + "r2": 0.7358851432800293, + "score": -0.20332438513408257 + }, + "val": { + "rmse": 0.233840053737128, + "mae": 0.12618696689605713, + "r2": 0.6347509622573853, + "score": -0.233840053737128 + }, + "test": { + "rmse": 0.23548819505307594, + "mae": 0.14876849949359894, + "r2": 0.6210535764694214, + "score": -0.23548819505307594 + } + }, + "time": "0:01:08.539078", + "diversity": { + "train": { + "mean_centered_corr": -0.016982607543468475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05261818319559097 + }, + "val": { + "mean_centered_corr": -0.017025893554091454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08680207282304764 + }, + "test": { + "mean_centered_corr": -0.01696028746664524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11552654951810837 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 27, + 25, + 19, + 8, + 4, + 28, + 17, + 15 + ], + "head_selection_time": "0:00:00.145633" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0877ee5ba --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-best-head-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2357054301684337, + "mae": 0.12653841078281403, + "r2": 0.6450614929199219, + "score": -0.2357054301684337 + }, + "val": { + "rmse": 0.23765212789668771, + "mae": 0.13472485542297363, + "r2": 0.6227452754974365, + "score": -0.23765212789668771 + }, + "test": { + "rmse": 0.24778267915496135, + "mae": 0.16990186274051666, + "r2": 0.5804521441459656, + "score": -0.24778267915496135 + } + }, + "time": "0:01:33.893743", + "diversity": { + "train": { + "mean_centered_corr": -0.018528519198298454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045790109783411026 + }, + "val": { + "mean_centered_corr": -0.01869109645485878, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057747576385736465 + }, + "test": { + "mean_centered_corr": -0.017511537298560143, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07643643766641617 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 18 + ], + "head_selection_time": "0:00:00.015586" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0.toml new file mode 100644 index 000000000..485d94624 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0/report.json new file mode 100644 index 000000000..644da59b3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2313646747453307, + "mae": 0.12118230015039444, + "r2": 0.6580142974853516, + "score": -0.2313646747453307 + }, + "val": { + "rmse": 0.2354418316334552, + "mae": 0.1258188933134079, + "r2": 0.6297299861907959, + "score": -0.2354418316334552 + }, + "test": { + "rmse": 0.23162903716515967, + "mae": 0.14173103868961334, + "r2": 0.6333720684051514, + "score": -0.23162903716515967 + } + }, + "time": "0:01:33.893743", + "diversity": { + "train": { + "mean_centered_corr": -0.018528519198298454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045790109783411026 + }, + "val": { + "mean_centered_corr": -0.01869109645485878, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057747576385736465 + }, + "test": { + "mean_centered_corr": -0.017511537298560143, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07643643766641617 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d9d2a13ce --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/smoke/sberbank-housing/rla_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23082056510008958, + "mae": 0.12217259407043457, + "r2": 0.6596208810806274, + "score": -0.23082056510008958 + }, + "val": { + "rmse": 0.2336025622310011, + "mae": 0.12522470951080322, + "r2": 0.6354925036430359, + "score": -0.2336025622310011 + }, + "test": { + "rmse": 0.23389241285301454, + "mae": 0.14725922048091888, + "r2": 0.6261719465255737, + "score": -0.23389241285301454 + } + }, + "time": "0:01:33.893743", + "diversity": { + "train": { + "mean_centered_corr": -0.018528519198298454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045790109783411026 + }, + "val": { + "mean_centered_corr": -0.01869109645485878, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057747576385736465 + }, + "test": { + "mean_centered_corr": -0.017511537298560143, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07643643766641617 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 18, + 21, + 19, + 25, + 6, + 4 + ], + "head_selection_time": "0:00:00.102958" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..de85e5c86 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-best-head-evaluation/0/report.json @@ -0,0 +1,455 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4515053148590503, + "mae": 0.35153692960739136, + "r2": 0.5317407250404358, + "score": -0.4515053148590503 + }, + "val": { + "rmse": 0.46391938568359065, + "mae": 0.36074763536453247, + "r2": 0.5064277052879333, + "score": -0.46391938568359065 + }, + "test": { + "rmse": 0.48142889277254225, + "mae": 0.3758453130722046, + "r2": 0.4832448363304138, + "score": -0.48142889277254225 + } + }, + "time": "0:38:21.569603", + "diversity": { + "train": { + "mean_centered_corr": -0.009637685492634773, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04498574882745743 + }, + "val": { + "mean_centered_corr": -0.009298564866185188, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046711746603250504 + }, + "test": { + "mean_centered_corr": -0.00908930879086256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04780808091163635 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.028483" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..7de7ee9ce --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0.toml @@ -0,0 +1,74 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..e3ee653ca --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/0/report.json @@ -0,0 +1,451 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44819067366049176, + "mae": 0.3488732874393463, + "r2": 0.5385907888412476, + "score": -0.44819067366049176 + }, + "val": { + "rmse": 0.463355012524859, + "mae": 0.3601790964603424, + "r2": 0.5076278448104858, + "score": -0.463355012524859 + }, + "test": { + "rmse": 0.4809202538385541, + "mae": 0.3753686547279358, + "r2": 0.48433613777160645, + "score": -0.4809202538385541 + } + }, + "time": "0:38:21.569603", + "diversity": { + "train": { + "mean_centered_corr": -0.009637685492634773, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04498574882745743 + }, + "val": { + "mean_centered_corr": -0.009298564866185188, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046711746603250504 + }, + "test": { + "mean_centered_corr": -0.00908930879086256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04780808091163635 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..19d838edb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,464 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44759809638897813, + "mae": 0.3485293388366699, + "r2": 0.539810061454773, + "score": -0.44759809638897813 + }, + "val": { + "rmse": 0.4629352081332738, + "mae": 0.35992681980133057, + "r2": 0.5085196495056152, + "score": -0.4629352081332738 + }, + "test": { + "rmse": 0.4802514336856013, + "mae": 0.3748669922351837, + "r2": 0.48576945066452026, + "score": -0.4802514336856013 + } + }, + "time": "0:38:21.569603", + "diversity": { + "train": { + "mean_centered_corr": -0.009637685492634773, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04498574882745743 + }, + "val": { + "mean_centered_corr": -0.009298564866185188, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046711746603250504 + }, + "test": { + "mean_centered_corr": -0.00908930879086256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04780808091163635 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 25, + 24, + 19, + 1, + 21, + 27, + 15, + 2, + 12, + 31 + ], + "head_selection_time": "0:00:00.370342" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..82f9fc957 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,202 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44844456178640835, + "mae": 0.34928542375564575, + "r2": 0.5380678772926331, + "score": -0.44844456178640835 + }, + "val": { + "rmse": 0.4649621508176366, + "mae": 0.3616116940975189, + "r2": 0.5042064189910889, + "score": -0.4649621508176366 + }, + "test": { + "rmse": 0.48187581714110606, + "mae": 0.3764643371105194, + "r2": 0.48228490352630615, + "score": -0.48187581714110606 + } + }, + "time": "0:33:40.844452", + "diversity": { + "train": { + "mean_centered_corr": -0.020041001960635185, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04958989471197128 + }, + "val": { + "mean_centered_corr": -0.019671577960252762, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05151689797639847 + }, + "test": { + "mean_centered_corr": -0.019756164401769638, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052489347755908966 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 18 + ], + "head_selection_time": "0:00:00.028038" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..ccda6b804 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0.toml @@ -0,0 +1,63 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..dac75d544 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/0/report.json @@ -0,0 +1,198 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44612866614094726, + "mae": 0.3472161889076233, + "r2": 0.5428266525268555, + "score": -0.44612866614094726 + }, + "val": { + "rmse": 0.463067499804912, + "mae": 0.35988649725914, + "r2": 0.5082387328147888, + "score": -0.463067499804912 + }, + "test": { + "rmse": 0.4804862724768644, + "mae": 0.3750061094760895, + "r2": 0.48526638746261597, + "score": -0.4804862724768644 + } + }, + "time": "0:33:40.844452", + "diversity": { + "train": { + "mean_centered_corr": -0.020041001960635185, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04958989471197128 + }, + "val": { + "mean_centered_corr": -0.019671577960252762, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05151689797639847 + }, + "test": { + "mean_centered_corr": -0.019756164401769638, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052489347755908966 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..a026853c9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4463454196329557, + "mae": 0.3473359942436218, + "r2": 0.5423822999000549, + "score": -0.4463454196329557 + }, + "val": { + "rmse": 0.46276401446104365, + "mae": 0.3595953583717346, + "r2": 0.5088831186294556, + "score": -0.46276401446104365 + }, + "test": { + "rmse": 0.4800152693247022, + "mae": 0.37460875511169434, + "r2": 0.4862750768661499, + "score": -0.4800152693247022 + } + }, + "time": "0:33:40.844452", + "diversity": { + "train": { + "mean_centered_corr": -0.020041001960635185, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04958989471197128 + }, + "val": { + "mean_centered_corr": -0.019671577960252762, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05151689797639847 + }, + "test": { + "mean_centered_corr": -0.019756164401769638, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052489347755908966 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 18, + 19, + 25, + 14, + 26, + 11, + 3, + 27, + 16, + 5, + 12, + 21, + 29, + 14, + 26, + 24, + 25, + 19 + ], + "head_selection_time": "0:00:00.765242" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-best-head-evaluation/0/report.json new file mode 100644 index 000000000..82580a9ac --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-best-head-evaluation/0/report.json @@ -0,0 +1,445 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.45139719963574126, + "mae": 0.35117724537849426, + "r2": 0.5319649577140808, + "score": -0.45139719963574126 + }, + "val": { + "rmse": 0.46397896473985684, + "mae": 0.36049172282218933, + "r2": 0.5063009262084961, + "score": -0.46397896473985684 + }, + "test": { + "rmse": 0.481064806640852, + "mae": 0.3752272427082062, + "r2": 0.48402613401412964, + "score": -0.481064806640852 + } + }, + "time": "0:23:29.845146", + "diversity": { + "train": { + "mean_centered_corr": -0.009395277127623558, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045357972383499146 + }, + "val": { + "mean_centered_corr": -0.008926574140787125, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04711531847715378 + }, + "test": { + "mean_centered_corr": -0.009024984203279018, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04819539934396744 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.027833" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0.toml new file mode 100644 index 000000000..fbe196209 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0.toml @@ -0,0 +1,64 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0/report.json new file mode 100644 index 000000000..30473de49 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-evaluation/0/report.json @@ -0,0 +1,441 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44808515070926874, + "mae": 0.3487781286239624, + "r2": 0.5388079881668091, + "score": -0.44808515070926874 + }, + "val": { + "rmse": 0.4633378873945458, + "mae": 0.3601687252521515, + "r2": 0.5076642632484436, + "score": -0.4633378873945458 + }, + "test": { + "rmse": 0.48088029743109706, + "mae": 0.37534818053245544, + "r2": 0.48442184925079346, + "score": -0.48088029743109706 + } + }, + "time": "0:23:29.845146", + "diversity": { + "train": { + "mean_centered_corr": -0.009395277127623558, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045357972383499146 + }, + "val": { + "mean_centered_corr": -0.008926574140787125, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04711531847715378 + }, + "test": { + "mean_centered_corr": -0.009024984203279018, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04819539934396744 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8f37ba891 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_esam-greedy-heads-evaluation/0/report.json @@ -0,0 +1,457 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4475514693736722, + "mae": 0.3484344184398651, + "r2": 0.5399059653282166, + "score": -0.4475514693736722 + }, + "val": { + "rmse": 0.4629167637909319, + "mae": 0.3598645031452179, + "r2": 0.5085588097572327, + "score": -0.4629167637909319 + }, + "test": { + "rmse": 0.4803221564231222, + "mae": 0.3749236762523651, + "r2": 0.485617995262146, + "score": -0.4803221564231222 + } + }, + "time": "0:23:29.845146", + "diversity": { + "train": { + "mean_centered_corr": -0.009395277127623558, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045357972383499146 + }, + "val": { + "mean_centered_corr": -0.008926574140787125, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04711531847715378 + }, + "test": { + "mean_centered_corr": -0.009024984203279018, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04819539934396744 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 24, + 27, + 19, + 2, + 12, + 25, + 1, + 11, + 31, + 14, + 27, + 2, + 15 + ], + "head_selection_time": "0:00:00.488491" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..de00f541b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,419 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.451333797182134, + "mae": 0.3511902987957001, + "r2": 0.5320963859558105, + "score": -0.451333797182134 + }, + "val": { + "rmse": 0.46380663024741176, + "mae": 0.36054760217666626, + "r2": 0.5066676139831543, + "score": -0.46380663024741176 + }, + "test": { + "rmse": 0.48139068107848576, + "mae": 0.37560245394706726, + "r2": 0.48332685232162476, + "score": -0.48139068107848576 + } + }, + "time": "0:35:13.729663", + "diversity": { + "train": { + "mean_centered_corr": -0.005216025747358799, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04148838296532631 + }, + "val": { + "mean_centered_corr": -0.005656818859279156, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0424356609582901 + }, + "test": { + "mean_centered_corr": -0.005289691034704447, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0431172214448452 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.028966" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0.toml new file mode 100644 index 000000000..f4e6abf99 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0.toml @@ -0,0 +1,61 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0/report.json new file mode 100644 index 000000000..e9f3634d6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/0/report.json @@ -0,0 +1,415 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44820339059913267, + "mae": 0.3488495647907257, + "r2": 0.5385645627975464, + "score": -0.44820339059913267 + }, + "val": { + "rmse": 0.46319753395446966, + "mae": 0.3600589334964752, + "r2": 0.5079624652862549, + "score": -0.46319753395446966 + }, + "test": { + "rmse": 0.48088071575887986, + "mae": 0.37532681226730347, + "r2": 0.4844209551811218, + "score": -0.48088071575887986 + } + }, + "time": "0:35:13.729663", + "diversity": { + "train": { + "mean_centered_corr": -0.005216025747358799, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04148838296532631 + }, + "val": { + "mean_centered_corr": -0.005656818859279156, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0424356609582901 + }, + "test": { + "mean_centered_corr": -0.005289691034704447, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0431172214448452 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..e82fce40f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,434 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4478290457374166, + "mae": 0.348653107881546, + "r2": 0.5393350124359131, + "score": -0.4478290457374166 + }, + "val": { + "rmse": 0.4628484521639483, + "mae": 0.3599395155906677, + "r2": 0.5087038278579712, + "score": -0.4628484521639483 + }, + "test": { + "rmse": 0.4805722784138239, + "mae": 0.37523025274276733, + "r2": 0.48508208990097046, + "score": -0.4805722784138239 + } + }, + "time": "0:35:13.729663", + "diversity": { + "train": { + "mean_centered_corr": -0.005216025747358799, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04148838296532631 + }, + "val": { + "mean_centered_corr": -0.005656818859279156, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0424356609582901 + }, + "test": { + "mean_centered_corr": -0.005289691034704447, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0431172214448452 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 24, + 14, + 2, + 31, + 22, + 21, + 25, + 14, + 2, + 19, + 27, + 31, + 12, + 2, + 14, + 3 + ], + "head_selection_time": "0:00:00.651502" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9055fe557 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-best-head-evaluation/0/report.json @@ -0,0 +1,446 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.45204026182588536, + "mae": 0.3519219160079956, + "r2": 0.5306304693222046, + "score": -0.45204026182588536 + }, + "val": { + "rmse": 0.4640838759408798, + "mae": 0.36102116107940674, + "r2": 0.5060776472091675, + "score": -0.4640838759408798 + }, + "test": { + "rmse": 0.48156976542107044, + "mae": 0.37596550583839417, + "r2": 0.4829423427581787, + "score": -0.48156976542107044 + } + }, + "time": "0:20:21.935298", + "diversity": { + "train": { + "mean_centered_corr": -0.009009289555251598, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04566791653633118 + }, + "val": { + "mean_centered_corr": -0.008741805329918861, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04742306470870972 + }, + "test": { + "mean_centered_corr": -0.008637306280434132, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04842785745859146 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.029409" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0.toml new file mode 100644 index 000000000..cdf7f3e0f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0.toml @@ -0,0 +1,67 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0/report.json new file mode 100644 index 000000000..4cd78cf83 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/0/report.json @@ -0,0 +1,442 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4485826549798914, + "mae": 0.34918203949928284, + "r2": 0.5377833247184753, + "score": -0.4485826549798914 + }, + "val": { + "rmse": 0.4634391174751191, + "mae": 0.3602841794490814, + "r2": 0.5074490904808044, + "score": -0.4634391174751191 + }, + "test": { + "rmse": 0.4810216869909678, + "mae": 0.3754705488681793, + "r2": 0.48411864042282104, + "score": -0.4810216869909678 + } + }, + "time": "0:20:21.935298", + "diversity": { + "train": { + "mean_centered_corr": -0.009009289555251598, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04566791653633118 + }, + "val": { + "mean_centered_corr": -0.008741805329918861, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04742306470870972 + }, + "test": { + "mean_centered_corr": -0.008637306280434132, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04842785745859146 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..a875673e6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-greedy-heads-evaluation/0/report.json @@ -0,0 +1,461 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44756392147614626, + "mae": 0.3484703004360199, + "r2": 0.5398803949356079, + "score": -0.44756392147614626 + }, + "val": { + "rmse": 0.46292271885090186, + "mae": 0.3598572313785553, + "r2": 0.5085461735725403, + "score": -0.46292271885090186 + }, + "test": { + "rmse": 0.48053390557368486, + "mae": 0.3750901222229004, + "r2": 0.4851643443107605, + "score": -0.48053390557368486 + } + }, + "time": "0:20:21.935298", + "diversity": { + "train": { + "mean_centered_corr": -0.009009289555251598, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04566791653633118 + }, + "val": { + "mean_centered_corr": -0.008741805329918861, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04742306470870972 + }, + "test": { + "mean_centered_corr": -0.008637306280434132, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04842785745859146 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 25, + 27, + 15, + 2, + 13, + 24, + 21, + 31, + 19, + 1, + 11, + 12, + 27, + 15, + 2, + 24 + ], + "head_selection_time": "0:00:00.649511" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..222652a2b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-best-head-evaluation/0/report.json @@ -0,0 +1,400 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.45199979641686133, + "mae": 0.3517935872077942, + "r2": 0.5307145118713379, + "score": -0.45199979641686133 + }, + "val": { + "rmse": 0.4638214410008231, + "mae": 0.360493928194046, + "r2": 0.5066360831260681, + "score": -0.4638214410008231 + }, + "test": { + "rmse": 0.4819659035779245, + "mae": 0.37611493468284607, + "r2": 0.48209136724472046, + "score": -0.4819659035779245 + } + }, + "time": "0:10:10.466567", + "diversity": { + "train": { + "mean_centered_corr": -0.005496397614479065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04279099032282829 + }, + "val": { + "mean_centered_corr": -0.005842795129865408, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04372712969779968 + }, + "test": { + "mean_centered_corr": -0.005908838007599115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044375959783792496 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.027383" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0.toml new file mode 100644 index 000000000..784c17199 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0/report.json new file mode 100644 index 000000000..f58c2dd92 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0/report.json @@ -0,0 +1,396 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4482607702587857, + "mae": 0.34888917207717896, + "r2": 0.5384464263916016, + "score": -0.4482607702587857 + }, + "val": { + "rmse": 0.4632019090821483, + "mae": 0.3600703775882721, + "r2": 0.5079531669616699, + "score": -0.4632019090821483 + }, + "test": { + "rmse": 0.4808496810468957, + "mae": 0.3752937614917755, + "r2": 0.48448747396469116, + "score": -0.4808496810468957 + } + }, + "time": "0:10:10.466567", + "diversity": { + "train": { + "mean_centered_corr": -0.005496397614479065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04279099032282829 + }, + "val": { + "mean_centered_corr": -0.005842795129865408, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04372712969779968 + }, + "test": { + "mean_centered_corr": -0.005908838007599115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044375959783792496 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0f7f787a9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,413 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44782199154768965, + "mae": 0.34862202405929565, + "r2": 0.5393495559692383, + "score": -0.44782199154768965 + }, + "val": { + "rmse": 0.4628482268025946, + "mae": 0.35992974042892456, + "r2": 0.5087043046951294, + "score": -0.4628482268025946 + }, + "test": { + "rmse": 0.4803119031248436, + "mae": 0.3749502897262573, + "r2": 0.48563992977142334, + "score": -0.4803119031248436 + } + }, + "time": "0:10:10.466567", + "diversity": { + "train": { + "mean_centered_corr": -0.005496397614479065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04279099032282829 + }, + "val": { + "mean_centered_corr": -0.005842795129865408, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04372712969779968 + }, + "test": { + "mean_centered_corr": -0.005908838007599115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044375959783792496 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 27, + 14, + 2, + 21, + 25, + 2, + 12, + 22, + 14, + 27, + 2, + 19, + 24 + ], + "head_selection_time": "0:00:00.542336" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8007cdd2c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-best-head-evaluation/0/report.json @@ -0,0 +1,400 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.45293358344227996, + "mae": 0.35263803601264954, + "r2": 0.5287735462188721, + "score": -0.45293358344227996 + }, + "val": { + "rmse": 0.46397138529545745, + "mae": 0.3607122600078583, + "r2": 0.5063170194625854, + "score": -0.46397138529545745 + }, + "test": { + "rmse": 0.4813020969034895, + "mae": 0.3759235143661499, + "r2": 0.48351699113845825, + "score": -0.4813020969034895 + } + }, + "time": "0:10:10.580086", + "diversity": { + "train": { + "mean_centered_corr": -0.004884317051619291, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042778123170137405 + }, + "val": { + "mean_centered_corr": -0.005044749937951565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04371502622961998 + }, + "test": { + "mean_centered_corr": -0.005197949707508087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04438793286681175 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 2 + ], + "head_selection_time": "0:00:00.027731" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0.toml new file mode 100644 index 000000000..a22753f01 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0/report.json new file mode 100644 index 000000000..5f2deca4b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-evaluation/0/report.json @@ -0,0 +1,396 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4484508087097321, + "mae": 0.34904322028160095, + "r2": 0.5380549430847168, + "score": -0.4484508087097321 + }, + "val": { + "rmse": 0.4632809116304394, + "mae": 0.36012470722198486, + "r2": 0.5077853202819824, + "score": -0.4632809116304394 + }, + "test": { + "rmse": 0.48091704691326154, + "mae": 0.3753390908241272, + "r2": 0.4843430519104004, + "score": -0.48091704691326154 + } + }, + "time": "0:10:10.580086", + "diversity": { + "train": { + "mean_centered_corr": -0.004884317051619291, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042778123170137405 + }, + "val": { + "mean_centered_corr": -0.005044749937951565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04371502622961998 + }, + "test": { + "mean_centered_corr": -0.005197949707508087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04438793286681175 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7b442d96c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json @@ -0,0 +1,403 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44943641696524117, + "mae": 0.34985706210136414, + "r2": 0.5360222458839417, + "score": -0.44943641696524117 + }, + "val": { + "rmse": 0.4629530240908241, + "mae": 0.35998642444610596, + "r2": 0.5084818601608276, + "score": -0.4629530240908241 + }, + "test": { + "rmse": 0.480628211999089, + "mae": 0.37514716386795044, + "r2": 0.48496222496032715, + "score": -0.480628211999089 + } + }, + "time": "0:10:10.580086", + "diversity": { + "train": { + "mean_centered_corr": -0.004884317051619291, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042778123170137405 + }, + "val": { + "mean_centered_corr": -0.005044749937951565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04371502622961998 + }, + "test": { + "mean_centered_corr": -0.005197949707508087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04438793286681175 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 2, + 14, + 24, + 31 + ], + "head_selection_time": "0:00:00.141449" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3b365e9ce --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-best-head-evaluation/0/report.json @@ -0,0 +1,400 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 15096, + "metrics": { + "train": { + "rmse": 0.4517256240823104, + "mae": 0.35139286518096924, + "r2": 0.5312836170196533, + "score": -0.4517256240823104 + }, + "val": { + "rmse": 0.4641009735445309, + "mae": 0.36067935824394226, + "r2": 0.5060412287712097, + "score": -0.4641009735445309 + }, + "test": { + "rmse": 0.48193517064355434, + "mae": 0.3760278820991516, + "r2": 0.4821573495864868, + "score": -0.48193517064355434 + } + }, + "time": "0:08:57.117859", + "diversity": { + "train": { + "mean_centered_corr": -0.01122801098972559, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04195729270577431 + }, + "val": { + "mean_centered_corr": -0.010996237397193909, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04290297254920006 + }, + "test": { + "mean_centered_corr": -0.011064938269555569, + "mean_pairwise_disagreement": NaN, + "member_std": 0.043424952775239944 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.028283" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0.toml new file mode 100644 index 000000000..5cc98e607 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.2 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0/report.json new file mode 100644 index 000000000..6074e8c77 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-evaluation/0/report.json @@ -0,0 +1,396 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 15096, + "metrics": { + "train": { + "rmse": 0.44928435800197336, + "mae": 0.3496941924095154, + "r2": 0.5363361239433289, + "score": -0.44928435800197336 + }, + "val": { + "rmse": 0.4633341406854049, + "mae": 0.3601973056793213, + "r2": 0.5076721906661987, + "score": -0.4633341406854049 + }, + "test": { + "rmse": 0.4808806847716492, + "mae": 0.3753437399864197, + "r2": 0.4844210147857666, + "score": -0.4808806847716492 + } + }, + "time": "0:08:57.117859", + "diversity": { + "train": { + "mean_centered_corr": -0.01122801098972559, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04195729270577431 + }, + "val": { + "mean_centered_corr": -0.010996237397193909, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04290297254920006 + }, + "test": { + "mean_centered_corr": -0.011064938269555569, + "mean_pairwise_disagreement": NaN, + "member_std": 0.043424952775239944 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b663642e5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json @@ -0,0 +1,409 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 15096, + "metrics": { + "train": { + "rmse": 0.44893214163383177, + "mae": 0.3495011031627655, + "r2": 0.5370628237724304, + "score": -0.44893214163383177 + }, + "val": { + "rmse": 0.463044314080512, + "mae": 0.360007643699646, + "r2": 0.5082879662513733, + "score": -0.463044314080512 + }, + "test": { + "rmse": 0.4804926610439783, + "mae": 0.37509334087371826, + "r2": 0.4852526783943176, + "score": -0.4804926610439783 + } + }, + "time": "0:08:57.117859", + "diversity": { + "train": { + "mean_centered_corr": -0.01122801098972559, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04195729270577431 + }, + "val": { + "mean_centered_corr": -0.010996237397193909, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04290297254920006 + }, + "test": { + "mean_centered_corr": -0.011064938269555569, + "mean_pairwise_disagreement": NaN, + "member_std": 0.043424952775239944 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24, + 6, + 17, + 22, + 12, + 6, + 21, + 27, + 13, + 6 + ], + "head_selection_time": "0:00:00.354807" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..72d2eed76 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,410 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.45234552294131686, + "mae": 0.35200172662734985, + "r2": 0.5299963355064392, + "score": -0.45234552294131686 + }, + "val": { + "rmse": 0.4639245891190366, + "mae": 0.3605712652206421, + "r2": 0.5064166784286499, + "score": -0.4639245891190366 + }, + "test": { + "rmse": 0.48198567488759675, + "mae": 0.3760890066623688, + "r2": 0.48204880952835083, + "score": -0.48198567488759675 + } + }, + "time": "0:17:32.517766", + "diversity": { + "train": { + "mean_centered_corr": -0.004823723807930946, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042313702404499054 + }, + "val": { + "mean_centered_corr": -0.005078355316072702, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04322447255253792 + }, + "test": { + "mean_centered_corr": -0.005122650880366564, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04387650638818741 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.028152" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..53fd18ac6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0.toml @@ -0,0 +1,54 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..c93a16e5b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/0/report.json @@ -0,0 +1,406 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44859881540040863, + "mae": 0.34915608167648315, + "r2": 0.5377500057220459, + "score": -0.44859881540040863 + }, + "val": { + "rmse": 0.4632865725335359, + "mae": 0.3601624369621277, + "r2": 0.5077732801437378, + "score": -0.4632865725335359 + }, + "test": { + "rmse": 0.48093585437297304, + "mae": 0.37536507844924927, + "r2": 0.48430269956588745, + "score": -0.48093585437297304 + } + }, + "time": "0:17:32.517766", + "diversity": { + "train": { + "mean_centered_corr": -0.004823723807930946, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042313702404499054 + }, + "val": { + "mean_centered_corr": -0.005078355316072702, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04322447255253792 + }, + "test": { + "mean_centered_corr": -0.005122650880366564, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04387650638818741 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..21155c67f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,413 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44944187096507476, + "mae": 0.34989649057388306, + "r2": 0.5360109806060791, + "score": -0.44944187096507476 + }, + "val": { + "rmse": 0.4629679265241866, + "mae": 0.3600855767726898, + "r2": 0.5084501504898071, + "score": -0.4629679265241866 + }, + "test": { + "rmse": 0.4805148498004945, + "mae": 0.37517452239990234, + "r2": 0.48520517349243164, + "score": -0.4805148498004945 + } + }, + "time": "0:17:32.517766", + "diversity": { + "train": { + "mean_centered_corr": -0.004823723807930946, + "mean_pairwise_disagreement": NaN, + "member_std": 0.042313702404499054 + }, + "val": { + "mean_centered_corr": -0.005078355316072702, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04322447255253792 + }, + "test": { + "mean_centered_corr": -0.005122650880366564, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04387650638818741 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 14, + 2, + 22 + ], + "head_selection_time": "0:00:00.142880" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7207b878b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,193 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4493539523791639, + "mae": 0.35018929839134216, + "r2": 0.5361924767494202, + "score": -0.4493539523791639 + }, + "val": { + "rmse": 0.46541405002003006, + "mae": 0.36193493008613586, + "r2": 0.5032421946525574, + "score": -0.46541405002003006 + }, + "test": { + "rmse": 0.4825725860549557, + "mae": 0.3769952356815338, + "r2": 0.48078668117523193, + "score": -0.4825725860549557 + } + }, + "time": "0:19:04.620800", + "diversity": { + "train": { + "mean_centered_corr": -0.02012588083744049, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05346926301717758 + }, + "val": { + "mean_centered_corr": -0.01973523572087288, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0558118112385273 + }, + "test": { + "mean_centered_corr": -0.019737333059310913, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05675049126148224 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:00.027780" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0.toml new file mode 100644 index 000000000..dc867c4bc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0.toml @@ -0,0 +1,56 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0/report.json new file mode 100644 index 000000000..bc42a3cd8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/0/report.json @@ -0,0 +1,189 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4440070475639729, + "mae": 0.34553396701812744, + "r2": 0.5471645593643188, + "score": -0.4440070475639729 + }, + "val": { + "rmse": 0.4629259699594227, + "mae": 0.3597675561904907, + "r2": 0.5085393190383911, + "score": -0.4629259699594227 + }, + "test": { + "rmse": 0.4803877661460669, + "mae": 0.3748730719089508, + "r2": 0.4854774475097656, + "score": -0.4803877661460669 + } + }, + "time": "0:19:04.620800", + "diversity": { + "train": { + "mean_centered_corr": -0.02012588083744049, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05346926301717758 + }, + "val": { + "mean_centered_corr": -0.01973523572087288, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0558118112385273 + }, + "test": { + "mean_centered_corr": -0.019737333059310913, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05675049126148224 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..18d91dc2a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4441718165198698, + "mae": 0.3457135260105133, + "r2": 0.5468284487724304, + "score": -0.4441718165198698 + }, + "val": { + "rmse": 0.4625995219609569, + "mae": 0.3594971299171448, + "r2": 0.5092321634292603, + "score": -0.4625995219609569 + }, + "test": { + "rmse": 0.47992605869184196, + "mae": 0.37455591559410095, + "r2": 0.4864659905433655, + "score": -0.47992605869184196 + } + }, + "time": "0:19:04.620800", + "diversity": { + "train": { + "mean_centered_corr": -0.02012588083744049, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05346926301717758 + }, + "val": { + "mean_centered_corr": -0.01973523572087288, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0558118112385273 + }, + "test": { + "mean_centered_corr": -0.019737333059310913, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05675049126148224 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11, + 24, + 25, + 5, + 3, + 27, + 17, + 4, + 18, + 1, + 24, + 25, + 20, + 8, + 14, + 30 + ], + "head_selection_time": "0:00:00.669894" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ad4af5dc8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-best-head-evaluation/0/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44680097340283104, + "mae": 0.3478912115097046, + "r2": 0.5414477586746216, + "score": -0.44680097340283104 + }, + "val": { + "rmse": 0.4650444270773418, + "mae": 0.36133578419685364, + "r2": 0.5040308833122253, + "score": -0.4650444270773418 + }, + "test": { + "rmse": 0.48297671022399175, + "mae": 0.37703683972358704, + "r2": 0.47991669178009033, + "score": -0.48297671022399175 + } + }, + "time": "0:15:51.200944", + "diversity": { + "train": { + "mean_centered_corr": -0.020390046760439873, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05101626738905907 + }, + "val": { + "mean_centered_corr": -0.02039702609181404, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052604030817747116 + }, + "test": { + "mean_centered_corr": -0.020311973989009857, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05323263257741928 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 17 + ], + "head_selection_time": "0:00:00.029762" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0.toml new file mode 100644 index 000000000..b1552956d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0/report.json new file mode 100644 index 000000000..d933df272 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/0/report.json @@ -0,0 +1,153 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.443936564623215, + "mae": 0.3454599678516388, + "r2": 0.5473083257675171, + "score": -0.443936564623215 + }, + "val": { + "rmse": 0.46276101981921397, + "mae": 0.35961103439331055, + "r2": 0.5088894367218018, + "score": -0.46276101981921397 + }, + "test": { + "rmse": 0.4801992110036178, + "mae": 0.3746589124202728, + "r2": 0.4858812689781189, + "score": -0.4801992110036178 + } + }, + "time": "0:15:51.200944", + "diversity": { + "train": { + "mean_centered_corr": -0.020390046760439873, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05101626738905907 + }, + "val": { + "mean_centered_corr": -0.02039702609181404, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052604030817747116 + }, + "test": { + "mean_centered_corr": -0.020311973989009857, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05323263257741928 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..67847cbdd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,168 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4438539175131199, + "mae": 0.3454529643058777, + "r2": 0.5474768877029419, + "score": -0.4438539175131199 + }, + "val": { + "rmse": 0.4625151676835406, + "mae": 0.35954388976097107, + "r2": 0.5094111561775208, + "score": -0.4625151676835406 + }, + "test": { + "rmse": 0.479742369480289, + "mae": 0.37443631887435913, + "r2": 0.4868590235710144, + "score": -0.479742369480289 + } + }, + "time": "0:15:51.200944", + "diversity": { + "train": { + "mean_centered_corr": -0.020390046760439873, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05101626738905907 + }, + "val": { + "mean_centered_corr": -0.02039702609181404, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052604030817747116 + }, + "test": { + "mean_centered_corr": -0.020311973989009857, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05323263257741928 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 17, + 27, + 3, + 5, + 4, + 10, + 14, + 24, + 25, + 20, + 16, + 22 + ], + "head_selection_time": "0:00:00.467943" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ef69ab721 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-best-head-evaluation/0/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44801757104722606, + "mae": 0.348697692155838, + "r2": 0.5389471054077148, + "score": -0.44801757104722606 + }, + "val": { + "rmse": 0.46513258351328474, + "mae": 0.36148601770401, + "r2": 0.503842830657959, + "score": -0.46513258351328474 + }, + "test": { + "rmse": 0.48253251942068065, + "mae": 0.37657663226127625, + "r2": 0.48087286949157715, + "score": -0.48253251942068065 + } + }, + "time": "0:15:54.055312", + "diversity": { + "train": { + "mean_centered_corr": -0.020338764414191246, + "mean_pairwise_disagreement": NaN, + "member_std": 0.050871677696704865 + }, + "val": { + "mean_centered_corr": -0.02030899003148079, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05245138704776764 + }, + "test": { + "mean_centered_corr": -0.020395034924149513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053137533366680145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 7 + ], + "head_selection_time": "0:00:00.028441" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0.toml new file mode 100644 index 000000000..1d55eda06 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0/report.json new file mode 100644 index 000000000..7b9d04d15 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0/report.json @@ -0,0 +1,153 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4439953515264817, + "mae": 0.34551990032196045, + "r2": 0.547188401222229, + "score": -0.4439953515264817 + }, + "val": { + "rmse": 0.4627563346148938, + "mae": 0.3595946729183197, + "r2": 0.5088993906974792, + "score": -0.4627563346148938 + }, + "test": { + "rmse": 0.4802735094812688, + "mae": 0.3747326135635376, + "r2": 0.4857221841812134, + "score": -0.4802735094812688 + } + }, + "time": "0:15:54.055312", + "diversity": { + "train": { + "mean_centered_corr": -0.020338764414191246, + "mean_pairwise_disagreement": NaN, + "member_std": 0.050871677696704865 + }, + "val": { + "mean_centered_corr": -0.02030899003148079, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05245138704776764 + }, + "test": { + "mean_centered_corr": -0.020395034924149513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053137533366680145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d892e337e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4440075509733049, + "mae": 0.345533162355423, + "r2": 0.5471635460853577, + "score": -0.4440075509733049 + }, + "val": { + "rmse": 0.46246732196836127, + "mae": 0.35945001244544983, + "r2": 0.5095126032829285, + "score": -0.46246732196836127 + }, + "test": { + "rmse": 0.4800169922135662, + "mae": 0.3746195435523987, + "r2": 0.48627138137817383, + "score": -0.4800169922135662 + } + }, + "time": "0:15:54.055312", + "diversity": { + "train": { + "mean_centered_corr": -0.020338764414191246, + "mean_pairwise_disagreement": NaN, + "member_std": 0.050871677696704865 + }, + "val": { + "mean_centered_corr": -0.02030899003148079, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05245138704776764 + }, + "test": { + "mean_centered_corr": -0.020395034924149513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053137533366680145 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 7, + 5, + 27, + 17, + 24, + 3, + 14, + 4, + 22, + 5, + 10, + 21, + 17, + 27, + 24, + 16, + 3, + 5, + 12, + 18, + 14, + 27, + 17 + ], + "head_selection_time": "0:00:01.091181" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c87dbd7e3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-best-head-evaluation/0/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4481166090239535, + "mae": 0.3489066958427429, + "r2": 0.538743257522583, + "score": -0.4481166090239535 + }, + "val": { + "rmse": 0.4651283707110853, + "mae": 0.3615894913673401, + "r2": 0.5038518309593201, + "score": -0.4651283707110853 + }, + "test": { + "rmse": 0.4823068025931435, + "mae": 0.37674346566200256, + "r2": 0.48135846853256226, + "score": -0.4823068025931435 + } + }, + "time": "0:15:51.054198", + "diversity": { + "train": { + "mean_centered_corr": -0.020383337512612343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05032477155327797 + }, + "val": { + "mean_centered_corr": -0.02031499706208706, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051892586052417755 + }, + "test": { + "mean_centered_corr": -0.02030996046960354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05258215218782425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 3 + ], + "head_selection_time": "0:00:00.029198" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0.toml new file mode 100644 index 000000000..ae52f3ec3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0/report.json new file mode 100644 index 000000000..47f9c456b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/0/report.json @@ -0,0 +1,153 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44408822333443154, + "mae": 0.3455769121646881, + "r2": 0.5469989776611328, + "score": -0.44408822333443154 + }, + "val": { + "rmse": 0.4628077726618535, + "mae": 0.3596377372741699, + "r2": 0.5087901949882507, + "score": -0.4628077726618535 + }, + "test": { + "rmse": 0.48022789849772834, + "mae": 0.37472617626190186, + "r2": 0.48581981658935547, + "score": -0.48022789849772834 + } + }, + "time": "0:15:51.054198", + "diversity": { + "train": { + "mean_centered_corr": -0.020383337512612343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05032477155327797 + }, + "val": { + "mean_centered_corr": -0.02031499706208706, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051892586052417755 + }, + "test": { + "mean_centered_corr": -0.02030996046960354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05258215218782425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4ca5e3f0e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,167 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4439705320848652, + "mae": 0.34557339549064636, + "r2": 0.5472390651702881, + "score": -0.4439705320848652 + }, + "val": { + "rmse": 0.4625398779765968, + "mae": 0.35953423380851746, + "r2": 0.5093587040901184, + "score": -0.4625398779765968 + }, + "test": { + "rmse": 0.4797024392153251, + "mae": 0.3744461238384247, + "r2": 0.48694443702697754, + "score": -0.4797024392153251 + } + }, + "time": "0:15:51.054198", + "diversity": { + "train": { + "mean_centered_corr": -0.020383337512612343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05032477155327797 + }, + "val": { + "mean_centered_corr": -0.02031499706208706, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051892586052417755 + }, + "test": { + "mean_centered_corr": -0.02030996046960354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05258215218782425 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 3, + 17, + 5, + 27, + 22, + 14, + 10, + 4, + 24, + 25, + 20 + ], + "head_selection_time": "0:00:00.400838" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b24f0e085 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-best-head-evaluation/0/report.json @@ -0,0 +1,157 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44671700492269856, + "mae": 0.3479386866092682, + "r2": 0.5416200160980225, + "score": -0.44671700492269856 + }, + "val": { + "rmse": 0.46502524927765376, + "mae": 0.36198684573173523, + "r2": 0.504071831703186, + "score": -0.46502524927765376 + }, + "test": { + "rmse": 0.48171681453527604, + "mae": 0.3763014078140259, + "r2": 0.4826264977455139, + "score": -0.48171681453527604 + } + }, + "time": "0:15:52.573639", + "diversity": { + "train": { + "mean_centered_corr": -0.020365934818983078, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049931272864341736 + }, + "val": { + "mean_centered_corr": -0.020258555188775063, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05144479125738144 + }, + "test": { + "mean_centered_corr": -0.020444629713892937, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05212138593196869 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.028008" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0.toml new file mode 100644 index 000000000..01868a998 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.01 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0/report.json new file mode 100644 index 000000000..ec2147338 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/0/report.json @@ -0,0 +1,153 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4441836085526797, + "mae": 0.3456646800041199, + "r2": 0.5468044281005859, + "score": -0.4441836085526797 + }, + "val": { + "rmse": 0.462806194991412, + "mae": 0.3596384525299072, + "r2": 0.5087935924530029, + "score": -0.462806194991412 + }, + "test": { + "rmse": 0.4802449022823802, + "mae": 0.3747553825378418, + "r2": 0.4857834577560425, + "score": -0.4802449022823802 + } + }, + "time": "0:15:52.573639", + "diversity": { + "train": { + "mean_centered_corr": -0.020365934818983078, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049931272864341736 + }, + "val": { + "mean_centered_corr": -0.020258555188775063, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05144479125738144 + }, + "test": { + "mean_centered_corr": -0.020444629713892937, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05212138593196869 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d77e052f8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,172 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4441221456449407, + "mae": 0.345669686794281, + "r2": 0.5469298362731934, + "score": -0.4441221456449407 + }, + "val": { + "rmse": 0.46260330683144324, + "mae": 0.35956308245658875, + "r2": 0.5092241168022156, + "score": -0.46260330683144324 + }, + "test": { + "rmse": 0.47994838230926673, + "mae": 0.37468981742858887, + "r2": 0.4864181876182556, + "score": -0.47994838230926673 + } + }, + "time": "0:15:52.573639", + "diversity": { + "train": { + "mean_centered_corr": -0.020365934818983078, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049931272864341736 + }, + "val": { + "mean_centered_corr": -0.020258555188775063, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05144479125738144 + }, + "test": { + "mean_centered_corr": -0.020444629713892937, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05212138593196869 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 14, + 3, + 10, + 5, + 15, + 17, + 27, + 25, + 20, + 22, + 24, + 14, + 10, + 21, + 3 + ], + "head_selection_time": "0:00:00.639394" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4d1fd8128 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,436 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.45186522081485614, + "mae": 0.35178253054618835, + "r2": 0.5309939384460449, + "score": -0.45186522081485614 + }, + "val": { + "rmse": 0.46410829401604725, + "mae": 0.36094433069229126, + "r2": 0.5060256719589233, + "score": -0.46410829401604725 + }, + "test": { + "rmse": 0.4815192949196226, + "mae": 0.3759884238243103, + "r2": 0.48305070400238037, + "score": -0.4815192949196226 + } + }, + "time": "0:12:01.248416", + "diversity": { + "train": { + "mean_centered_corr": -0.008978656493127346, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04612109065055847 + }, + "val": { + "mean_centered_corr": -0.008577198721468449, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04788525402545929 + }, + "test": { + "mean_centered_corr": -0.008744378574192524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04893070459365845 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.028774" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..5bd9cae8a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0.toml @@ -0,0 +1,57 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..071bb8a22 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-evaluation/0/report.json @@ -0,0 +1,432 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44850098031020863, + "mae": 0.3491051197052002, + "r2": 0.5379515886306763, + "score": -0.44850098031020863 + }, + "val": { + "rmse": 0.4633989882463895, + "mae": 0.3602413833141327, + "r2": 0.507534384727478, + "score": -0.4633989882463895 + }, + "test": { + "rmse": 0.4810100390661497, + "mae": 0.3754710555076599, + "r2": 0.48414361476898193, + "score": -0.4810100390661497 + } + }, + "time": "0:12:01.248416", + "diversity": { + "train": { + "mean_centered_corr": -0.008978656493127346, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04612109065055847 + }, + "val": { + "mean_centered_corr": -0.008577198721468449, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04788525402545929 + }, + "test": { + "mean_centered_corr": -0.008744378574192524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04893070459365845 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ff9be6afe --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,449 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4473229583907692, + "mae": 0.3482147455215454, + "r2": 0.5403756499290466, + "score": -0.4473229583907692 + }, + "val": { + "rmse": 0.46284785656584665, + "mae": 0.35979634523391724, + "r2": 0.5087051391601562, + "score": -0.46284785656584665 + }, + "test": { + "rmse": 0.4803782742299631, + "mae": 0.374970942735672, + "r2": 0.48549777269363403, + "score": -0.4803782742299631 + } + }, + "time": "0:12:01.248416", + "diversity": { + "train": { + "mean_centered_corr": -0.008978656493127346, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04612109065055847 + }, + "val": { + "mean_centered_corr": -0.008577198721468449, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04788525402545929 + }, + "test": { + "mean_centered_corr": -0.008744378574192524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04893070459365845 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 25, + 2, + 19, + 26, + 12, + 27, + 14, + 1, + 21, + 26, + 24, + 19, + 2, + 11 + ], + "head_selection_time": "0:00:00.537217" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-best-head-evaluation/0/report.json new file mode 100644 index 000000000..40a70ff9b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-best-head-evaluation/0/report.json @@ -0,0 +1,183 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.70361328125, + "min_features_per_member": 121.0, + "mean_features_per_member": 135.09375, + "max_features_per_member": 148.0, + "min_members_per_feature": 16.0, + "mean_members_per_feature": 22.515625, + "max_members_per_feature": 28.0, + "mean_pairwise_jaccard": 0.5425720205382745, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4475517856746811, + "mae": 0.34862762689590454, + "r2": 0.539905309677124, + "score": -0.4475517856746811 + }, + "val": { + "rmse": 0.46477365365967316, + "mae": 0.3614905774593353, + "r2": 0.5046082735061646, + "score": -0.46477365365967316 + }, + "test": { + "rmse": 0.48197477680705497, + "mae": 0.37628859281539917, + "r2": 0.48207223415374756, + "score": -0.48197477680705497 + } + }, + "time": "0:12:08.409861", + "diversity": { + "train": { + "mean_centered_corr": -0.019095098599791527, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06140593811869621 + }, + "val": { + "mean_centered_corr": -0.017920419573783875, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06572531163692474 + }, + "test": { + "mean_centered_corr": -0.017684392631053925, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06748230010271072 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.027824" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0.toml new file mode 100644 index 000000000..669bb05cd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.7 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0/report.json new file mode 100644 index 000000000..f5894fa36 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.70361328125, + "min_features_per_member": 121.0, + "mean_features_per_member": 135.09375, + "max_features_per_member": 148.0, + "min_members_per_feature": 16.0, + "mean_members_per_feature": 22.515625, + "max_members_per_feature": 28.0, + "mean_pairwise_jaccard": 0.5425720205382745, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44401042039559846, + "mae": 0.34563449025154114, + "r2": 0.5471577048301697, + "score": -0.44401042039559846 + }, + "val": { + "rmse": 0.4641162243965573, + "mae": 0.3610197603702545, + "r2": 0.5060087442398071, + "score": -0.4641162243965573 + }, + "test": { + "rmse": 0.4815966695140456, + "mae": 0.3760018050670624, + "r2": 0.48288458585739136, + "score": -0.4815966695140456 + } + }, + "time": "0:12:08.409861", + "diversity": { + "train": { + "mean_centered_corr": -0.019095098599791527, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06140593811869621 + }, + "val": { + "mean_centered_corr": -0.017920419573783875, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06572531163692474 + }, + "test": { + "mean_centered_corr": -0.017684392631053925, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06748230010271072 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3a48b73af --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.7-greedy-heads-evaluation/0/report.json @@ -0,0 +1,197 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.70361328125, + "min_features_per_member": 121.0, + "mean_features_per_member": 135.09375, + "max_features_per_member": 148.0, + "min_members_per_feature": 16.0, + "mean_members_per_feature": 22.515625, + "max_members_per_feature": 28.0, + "mean_pairwise_jaccard": 0.5425720205382745, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44399968094212866, + "mae": 0.3455778956413269, + "r2": 0.547179639339447, + "score": -0.44399968094212866 + }, + "val": { + "rmse": 0.46249551448708903, + "mae": 0.35950425267219543, + "r2": 0.5094528198242188, + "score": -0.46249551448708903 + }, + "test": { + "rmse": 0.47980768580603567, + "mae": 0.37452432513237, + "r2": 0.4867192506790161, + "score": -0.47980768580603567 + } + }, + "time": "0:12:08.409861", + "diversity": { + "train": { + "mean_centered_corr": -0.019095098599791527, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06140593811869621 + }, + "val": { + "mean_centered_corr": -0.017920419573783875, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06572531163692474 + }, + "test": { + "mean_centered_corr": -0.017684392631053925, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06748230010271072 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 4, + 7, + 24, + 25, + 11, + 9, + 31, + 12, + 27, + 14, + 22, + 24, + 5, + 20, + 25 + ], + "head_selection_time": "0:00:00.591913" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-best-head-evaluation/0/report.json new file mode 100644 index 000000000..50ed2f8b2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-best-head-evaluation/0/report.json @@ -0,0 +1,183 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8019205927848816, + "min_features_per_member": 141.0, + "mean_features_per_member": 153.96875, + "max_features_per_member": 167.0, + "min_members_per_feature": 21.0, + "mean_members_per_feature": 25.66145896911621, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6682771134235742, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44690438220657625, + "mae": 0.3479042053222656, + "r2": 0.5412354469299316, + "score": -0.44690438220657625 + }, + "val": { + "rmse": 0.4652213638497702, + "mae": 0.36172589659690857, + "r2": 0.5036534070968628, + "score": -0.4652213638497702 + }, + "test": { + "rmse": 0.48343941871247514, + "mae": 0.37721583247184753, + "r2": 0.4789196848869324, + "score": -0.48343941871247514 + } + }, + "time": "0:12:07.675037", + "diversity": { + "train": { + "mean_centered_corr": -0.019306395202875137, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057879555970430374 + }, + "val": { + "mean_centered_corr": -0.01832164078950882, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06112360581755638 + }, + "test": { + "mean_centered_corr": -0.01794510707259178, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06266573071479797 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.027886" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0.toml new file mode 100644 index 000000000..83cf7dea9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.8 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0/report.json new file mode 100644 index 000000000..835fe9a93 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8019205927848816, + "min_features_per_member": 141.0, + "mean_features_per_member": 153.96875, + "max_features_per_member": 167.0, + "min_members_per_feature": 21.0, + "mean_members_per_feature": 25.66145896911621, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6682771134235742, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4439757847261177, + "mae": 0.34554752707481384, + "r2": 0.5472283363342285, + "score": -0.4439757847261177 + }, + "val": { + "rmse": 0.4634085546239816, + "mae": 0.36031344532966614, + "r2": 0.5075141191482544, + "score": -0.4634085546239816 + }, + "test": { + "rmse": 0.4808915146871742, + "mae": 0.37534573674201965, + "r2": 0.4843977689743042, + "score": -0.4808915146871742 + } + }, + "time": "0:12:07.675037", + "diversity": { + "train": { + "mean_centered_corr": -0.019306395202875137, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057879555970430374 + }, + "val": { + "mean_centered_corr": -0.01832164078950882, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06112360581755638 + }, + "test": { + "mean_centered_corr": -0.01794510707259178, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06266573071479797 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c9f5dc71a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.8-greedy-heads-evaluation/0/report.json @@ -0,0 +1,193 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8019205927848816, + "min_features_per_member": 141.0, + "mean_features_per_member": 153.96875, + "max_features_per_member": 167.0, + "min_members_per_feature": 21.0, + "mean_members_per_feature": 25.66145896911621, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6682771134235742, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44390037903332324, + "mae": 0.34550046920776367, + "r2": 0.547382116317749, + "score": -0.44390037903332324 + }, + "val": { + "rmse": 0.4624822561623469, + "mae": 0.35955899953842163, + "r2": 0.5094809532165527, + "score": -0.4624822561623469 + }, + "test": { + "rmse": 0.4798057758257153, + "mae": 0.37443193793296814, + "r2": 0.4867233633995056, + "score": -0.4798057758257153 + } + }, + "time": "0:12:07.675037", + "diversity": { + "train": { + "mean_centered_corr": -0.019306395202875137, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057879555970430374 + }, + "val": { + "mean_centered_corr": -0.01832164078950882, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06112360581755638 + }, + "test": { + "mean_centered_corr": -0.01794510707259178, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06266573071479797 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 19, + 27, + 17, + 24, + 25, + 20, + 22, + 4, + 14, + 7, + 5 + ], + "head_selection_time": "0:00:00.407380" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b5e7724cd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-best-head-evaluation/0/report.json @@ -0,0 +1,183 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44680861066629424, + "mae": 0.3479525148868561, + "r2": 0.5414320230484009, + "score": -0.44680861066629424 + }, + "val": { + "rmse": 0.465403548325863, + "mae": 0.3621026575565338, + "r2": 0.5032646059989929, + "score": -0.465403548325863 + }, + "test": { + "rmse": 0.48311284423774803, + "mae": 0.377098023891449, + "r2": 0.4796234369277954, + "score": -0.48311284423774803 + } + }, + "time": "0:12:04.046616", + "diversity": { + "train": { + "mean_centered_corr": -0.020108899101614952, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05420300364494324 + }, + "val": { + "mean_centered_corr": -0.01979835145175457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05661233514547348 + }, + "test": { + "mean_centered_corr": -0.01982581429183483, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05757860466837883 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 17 + ], + "head_selection_time": "0:00:00.027733" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0.toml new file mode 100644 index 000000000..349df099f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0/report.json new file mode 100644 index 000000000..ecbe78388 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4438062088250124, + "mae": 0.34538015723228455, + "r2": 0.5475741624832153, + "score": -0.4438062088250124 + }, + "val": { + "rmse": 0.46290981075910853, + "mae": 0.3597574234008789, + "r2": 0.508573591709137, + "score": -0.46290981075910853 + }, + "test": { + "rmse": 0.480415480847424, + "mae": 0.37490591406822205, + "r2": 0.48541808128356934, + "score": -0.480415480847424 + } + }, + "time": "0:12:04.046616", + "diversity": { + "train": { + "mean_centered_corr": -0.020108899101614952, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05420300364494324 + }, + "val": { + "mean_centered_corr": -0.01979835145175457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05661233514547348 + }, + "test": { + "mean_centered_corr": -0.01982581429183483, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05757860466837883 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4888bef62 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.9-greedy-heads-evaluation/0/report.json @@ -0,0 +1,193 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4439127657129147, + "mae": 0.3454906642436981, + "r2": 0.5473568439483643, + "score": -0.4439127657129147 + }, + "val": { + "rmse": 0.4625593198933149, + "mae": 0.35958266258239746, + "r2": 0.5093174576759338, + "score": -0.4625593198933149 + }, + "test": { + "rmse": 0.479899744054174, + "mae": 0.37457266449928284, + "r2": 0.4865223169326782, + "score": -0.479899744054174 + } + }, + "time": "0:12:04.046616", + "diversity": { + "train": { + "mean_centered_corr": -0.020108899101614952, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05420300364494324 + }, + "val": { + "mean_centered_corr": -0.01979835145175457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05661233514547348 + }, + "test": { + "mean_centered_corr": -0.01982581429183483, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05757860466837883 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 17, + 14, + 11, + 24, + 25, + 5, + 4, + 27, + 20, + 1, + 9 + ], + "head_selection_time": "0:00:00.405579" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-best-head-evaluation/0/report.json new file mode 100644 index 000000000..08801eb88 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-best-head-evaluation/0/report.json @@ -0,0 +1,183 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9480794072151184, + "min_features_per_member": 176.0, + "mean_features_per_member": 182.03125, + "max_features_per_member": 188.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.33854103088379, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.900785048828594, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4475263477006594, + "mae": 0.34830242395401, + "r2": 0.5399576425552368, + "score": -0.4475263477006594 + }, + "val": { + "rmse": 0.4649031464912309, + "mae": 0.36123916506767273, + "r2": 0.5043321847915649, + "score": -0.4649031464912309 + }, + "test": { + "rmse": 0.4828450127182003, + "mae": 0.376803457736969, + "r2": 0.48020023107528687, + "score": -0.4828450127182003 + } + }, + "time": "0:12:04.148232", + "diversity": { + "train": { + "mean_centered_corr": -0.020304471254348755, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05282822996377945 + }, + "val": { + "mean_centered_corr": -0.02010956034064293, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0548769012093544 + }, + "test": { + "mean_centered_corr": -0.020047690719366074, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05574715510010719 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7 + ], + "head_selection_time": "0:00:00.027586" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0.toml new file mode 100644 index 000000000..3571f95da --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0/report.json new file mode 100644 index 000000000..f97797710 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9480794072151184, + "min_features_per_member": 176.0, + "mean_features_per_member": 182.03125, + "max_features_per_member": 188.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.33854103088379, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.900785048828594, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.443937454120516, + "mae": 0.34546512365341187, + "r2": 0.5473065376281738, + "score": -0.443937454120516 + }, + "val": { + "rmse": 0.46283832689198484, + "mae": 0.35965093970298767, + "r2": 0.5087253451347351, + "score": -0.46283832689198484 + }, + "test": { + "rmse": 0.48036774294670015, + "mae": 0.3747703433036804, + "r2": 0.48552030324935913, + "score": -0.48036774294670015 + } + }, + "time": "0:12:04.148232", + "diversity": { + "train": { + "mean_centered_corr": -0.020304471254348755, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05282822996377945 + }, + "val": { + "mean_centered_corr": -0.02010956034064293, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0548769012093544 + }, + "test": { + "mean_centered_corr": -0.020047690719366074, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05574715510010719 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..acdf05465 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/mfb_keep0.95-greedy-heads-evaluation/0/report.json @@ -0,0 +1,197 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1411040, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9480794072151184, + "min_features_per_member": 176.0, + "mean_features_per_member": 182.03125, + "max_features_per_member": 188.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.33854103088379, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.900785048828594, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2994 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 416 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.444191827565372, + "mae": 0.34573376178741455, + "r2": 0.5467876195907593, + "score": -0.444191827565372 + }, + "val": { + "rmse": 0.46250350473983365, + "mae": 0.35950493812561035, + "r2": 0.5094358921051025, + "score": -0.46250350473983365 + }, + "test": { + "rmse": 0.4799452154615134, + "mae": 0.37449464201927185, + "r2": 0.48642498254776, + "score": -0.4799452154615134 + } + }, + "time": "0:12:04.148232", + "diversity": { + "train": { + "mean_centered_corr": -0.020304471254348755, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05282822996377945 + }, + "val": { + "mean_centered_corr": -0.02010956034064293, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0548769012093544 + }, + "test": { + "mean_centered_corr": -0.020047690719366074, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05574715510010719 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7, + 14, + 17, + 20, + 3, + 24, + 22, + 1, + 5, + 9, + 28, + 14, + 11, + 17, + 24 + ], + "head_selection_time": "0:00:00.596709" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..439dc1c47 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,409 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.4519491065266232, + "mae": 0.3517553210258484, + "r2": 0.5308197736740112, + "score": -0.4519491065266232 + }, + "val": { + "rmse": 0.4638249749501882, + "mae": 0.36049750447273254, + "r2": 0.5066285729408264, + "score": -0.4638249749501882 + }, + "test": { + "rmse": 0.4819157837704579, + "mae": 0.3760508596897125, + "r2": 0.4821990132331848, + "score": -0.4819157837704579 + } + }, + "time": "0:21:02.492076", + "diversity": { + "train": { + "mean_centered_corr": -0.004915067460387945, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04214517027139664 + }, + "val": { + "mean_centered_corr": -0.00553104979917407, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04312119632959366 + }, + "test": { + "mean_centered_corr": -0.005271091125905514, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04374181479215622 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.030941" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..5233fa3a6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "cooking-time" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..d9a8d49f6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-evaluation/0/report.json @@ -0,0 +1,405 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44810794656884273, + "mae": 0.3487721383571625, + "r2": 0.5387611389160156, + "score": -0.44810794656884273 + }, + "val": { + "rmse": 0.46314488441622087, + "mae": 0.36001190543174744, + "r2": 0.5080743432044983, + "score": -0.46314488441622087 + }, + "test": { + "rmse": 0.48088551875549623, + "mae": 0.37529256939888, + "r2": 0.4844106435775757, + "score": -0.48088551875549623 + } + }, + "time": "0:21:02.492076", + "diversity": { + "train": { + "mean_centered_corr": -0.004915067460387945, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04214517027139664 + }, + "val": { + "mean_centered_corr": -0.00553104979917407, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04312119632959366 + }, + "test": { + "mean_centered_corr": -0.005271091125905514, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04374181479215622 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..788ac6a5d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,413 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "cooking-time", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 7, + 3, + 2, + 2, + 2, + 2 + ], + "n_features": 192, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/cooking-time" + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 17982, + "metrics": { + "train": { + "rmse": 0.44774267417605834, + "mae": 0.3486373722553253, + "r2": 0.5395127534866333, + "score": -0.44774267417605834 + }, + "val": { + "rmse": 0.4627888241622746, + "mae": 0.35998469591140747, + "r2": 0.5088304281234741, + "score": -0.4627888241622746 + }, + "test": { + "rmse": 0.480810230139191, + "mae": 0.3753969371318817, + "r2": 0.48457205295562744, + "score": -0.480810230139191 + } + }, + "time": "0:21:02.492076", + "diversity": { + "train": { + "mean_centered_corr": -0.004915067460387945, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04214517027139664 + }, + "val": { + "mean_centered_corr": -0.00553104979917407, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04312119632959366 + }, + "test": { + "mean_centered_corr": -0.005271091125905514, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04374181479215622 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 31, + 14, + 2, + 24, + 21 + ], + "head_selection_time": "0:00:00.181665" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f1c458896 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-best-head-evaluation/0/report.json @@ -0,0 +1,166 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44621279542701975, + "mae": 0.3473000228404999, + "r2": 0.5426542162895203, + "score": -0.44621279542701975 + }, + "val": { + "rmse": 0.46492255368357066, + "mae": 0.36141437292099, + "r2": 0.5042908191680908, + "score": -0.46492255368357066 + }, + "test": { + "rmse": 0.4825495500767423, + "mae": 0.37668848037719727, + "r2": 0.4808362126350403, + "score": -0.4825495500767423 + } + }, + "time": "0:37:31.095542", + "diversity": { + "train": { + "mean_centered_corr": -0.02036323957145214, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05055415257811546 + }, + "val": { + "mean_centered_corr": -0.02048061601817608, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05214421823620796 + }, + "test": { + "mean_centered_corr": -0.020323824137449265, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052737440913915634 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14 + ], + "head_selection_time": "0:00:00.027759" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0.toml new file mode 100644 index 000000000..0b28045f1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0/report.json new file mode 100644 index 000000000..9520ae39a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0/report.json @@ -0,0 +1,162 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4437324533888703, + "mae": 0.3453030586242676, + "r2": 0.5477245450019836, + "score": -0.4437324533888703 + }, + "val": { + "rmse": 0.4627353230636857, + "mae": 0.35955438017845154, + "r2": 0.5089439749717712, + "score": -0.4627353230636857 + }, + "test": { + "rmse": 0.4803077303885999, + "mae": 0.3747372627258301, + "r2": 0.48564887046813965, + "score": -0.4803077303885999 + } + }, + "time": "0:37:31.095542", + "diversity": { + "train": { + "mean_centered_corr": -0.02036323957145214, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05055415257811546 + }, + "val": { + "mean_centered_corr": -0.02048061601817608, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05214421823620796 + }, + "test": { + "mean_centered_corr": -0.020323824137449265, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052737440913915634 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..03c3c5cda --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-greedy-heads-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4439349534537547, + "mae": 0.3455897867679596, + "r2": 0.5473116636276245, + "score": -0.4439349534537547 + }, + "val": { + "rmse": 0.4624234187671886, + "mae": 0.3594515919685364, + "r2": 0.5096057653427124, + "score": -0.4624234187671886 + }, + "test": { + "rmse": 0.4800586966547807, + "mae": 0.37484052777290344, + "r2": 0.4861820936203003, + "score": -0.4800586966547807 + } + }, + "time": "0:37:31.095542", + "diversity": { + "train": { + "mean_centered_corr": -0.02036323957145214, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05055415257811546 + }, + "val": { + "mean_centered_corr": -0.02048061601817608, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05214421823620796 + }, + "test": { + "mean_centered_corr": -0.020323824137449265, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052737440913915634 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14, + 3, + 17, + 24, + 5, + 10, + 4, + 27, + 31, + 7, + 17, + 3, + 24 + ], + "head_selection_time": "0:00:00.494060" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..fd6ab34e6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,192 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44878779762176807, + "mae": 0.34966716170310974, + "r2": 0.5373604893684387, + "score": -0.44878779762176807 + }, + "val": { + "rmse": 0.46542866555060514, + "mae": 0.36194679141044617, + "r2": 0.5032110214233398, + "score": -0.46542866555060514 + }, + "test": { + "rmse": 0.48315113561982953, + "mae": 0.3771289885044098, + "r2": 0.47954094409942627, + "score": -0.48315113561982953 + } + }, + "time": "0:26:00.632881", + "diversity": { + "train": { + "mean_centered_corr": -0.02016231045126915, + "mean_pairwise_disagreement": NaN, + "member_std": 0.054188016802072525 + }, + "val": { + "mean_centered_corr": -0.019777867943048477, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05651267245411873 + }, + "test": { + "mean_centered_corr": -0.019633809104561806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057482462376356125 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 11 + ], + "head_selection_time": "0:00:00.028172" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0.toml new file mode 100644 index 000000000..704e95be9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0.toml @@ -0,0 +1,53 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0/report.json new file mode 100644 index 000000000..65cbfe41a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44355369712218456, + "mae": 0.34518396854400635, + "r2": 0.5480888485908508, + "score": -0.44355369712218456 + }, + "val": { + "rmse": 0.4628964355361592, + "mae": 0.3597174882888794, + "r2": 0.5086020231246948, + "score": -0.4628964355361592 + }, + "test": { + "rmse": 0.4804232040775468, + "mae": 0.3748829662799835, + "r2": 0.4854015111923218, + "score": -0.4804232040775468 + } + }, + "time": "0:26:00.632881", + "diversity": { + "train": { + "mean_centered_corr": -0.02016231045126915, + "mean_pairwise_disagreement": NaN, + "member_std": 0.054188016802072525 + }, + "val": { + "mean_centered_corr": -0.019777867943048477, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05651267245411873 + }, + "test": { + "mean_centered_corr": -0.019633809104561806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057482462376356125 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3a9a313cf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9000651240348816, + "min_features_per_member": 164.0, + "mean_features_per_member": 172.8125, + "max_features_per_member": 181.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.80208396911621, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8173701129083908, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44370132233029524, + "mae": 0.3453197777271271, + "r2": 0.5477880239486694, + "score": -0.44370132233029524 + }, + "val": { + "rmse": 0.46255113731378883, + "mae": 0.35945776104927063, + "r2": 0.5093348026275635, + "score": -0.46255113731378883 + }, + "test": { + "rmse": 0.4799569823688197, + "mae": 0.3745240867137909, + "r2": 0.4863998293876648, + "score": -0.4799569823688197 + } + }, + "time": "0:26:00.632881", + "diversity": { + "train": { + "mean_centered_corr": -0.02016231045126915, + "mean_pairwise_disagreement": NaN, + "member_std": 0.054188016802072525 + }, + "val": { + "mean_centered_corr": -0.019777867943048477, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05651267245411873 + }, + "test": { + "mean_centered_corr": -0.019633809104561806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057482462376356125 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 11, + 24, + 9, + 17, + 14, + 5, + 25, + 4, + 3, + 1, + 24, + 30, + 27, + 8, + 17, + 28 + ], + "head_selection_time": "0:00:00.648503" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9e3525e8a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1520160, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4464251521524547, + "mae": 0.347421795129776, + "r2": 0.542218804359436, + "score": -0.4464251521524547 + }, + "val": { + "rmse": 0.465058894016931, + "mae": 0.3617338240146637, + "r2": 0.5040000677108765, + "score": -0.465058894016931 + }, + "test": { + "rmse": 0.48247512338182996, + "mae": 0.3766128718852997, + "r2": 0.48099637031555176, + "score": -0.48247512338182996 + } + }, + "time": "0:14:45.597272", + "diversity": { + "train": { + "mean_centered_corr": -0.020377950742840767, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05120374634861946 + }, + "val": { + "mean_centered_corr": -0.02035089209675789, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052788279950618744 + }, + "test": { + "mean_centered_corr": -0.02027548849582672, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0534634031355381 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14 + ], + "head_selection_time": "0:00:00.027776" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..7ba625753 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..40a403eab --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-evaluation/0/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1520160, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4437718090554473, + "mae": 0.3453312814235687, + "r2": 0.5476442575454712, + "score": -0.4437718090554473 + }, + "val": { + "rmse": 0.4627581378606596, + "mae": 0.3596029281616211, + "r2": 0.5088955163955688, + "score": -0.4627581378606596 + }, + "test": { + "rmse": 0.4802975698569232, + "mae": 0.37475284934043884, + "r2": 0.48567062616348267, + "score": -0.4802975698569232 + } + }, + "time": "0:14:45.597272", + "diversity": { + "train": { + "mean_centered_corr": -0.020377950742840767, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05120374634861946 + }, + "val": { + "mean_centered_corr": -0.02035089209675789, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052788279950618744 + }, + "test": { + "mean_centered_corr": -0.02027548849582672, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0534634031355381 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0eb3076b7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,165 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1520160, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4437838635485539, + "mae": 0.34541016817092896, + "r2": 0.5476197004318237, + "score": -0.4437838635485539 + }, + "val": { + "rmse": 0.4624788086134533, + "mae": 0.3595048785209656, + "r2": 0.5094882845878601, + "score": -0.4624788086134533 + }, + "test": { + "rmse": 0.4800547545217561, + "mae": 0.3747299015522003, + "r2": 0.4861905574798584, + "score": -0.4800547545217561 + } + }, + "time": "0:14:45.597272", + "diversity": { + "train": { + "mean_centered_corr": -0.020377950742840767, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05120374634861946 + }, + "val": { + "mean_centered_corr": -0.02035089209675789, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052788279950618744 + }, + "test": { + "mean_centered_corr": -0.02027548849582672, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0534634031355381 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14, + 3, + 17, + 5, + 27, + 24, + 22, + 10, + 16, + 15 + ], + "head_selection_time": "0:00:00.358048" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5890a5a42 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1520160, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4462822343900781, + "mae": 0.347369521856308, + "r2": 0.5425118207931519, + "score": -0.4462822343900781 + }, + "val": { + "rmse": 0.465139054815724, + "mae": 0.36169102787971497, + "r2": 0.5038290619850159, + "score": -0.465139054815724 + }, + "test": { + "rmse": 0.48220568612119014, + "mae": 0.3766040503978729, + "r2": 0.4815758466720581, + "score": -0.48220568612119014 + } + }, + "time": "0:14:46.572437", + "diversity": { + "train": { + "mean_centered_corr": -0.02044508419930935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05110829323530197 + }, + "val": { + "mean_centered_corr": -0.020311282947659492, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05272478237748146 + }, + "test": { + "mean_centered_corr": -0.0204484686255455, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053339943289756775 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14 + ], + "head_selection_time": "0:00:00.028154" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..a07ce792d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..ba5555ef9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-evaluation/0/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1520160, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4437945578595596, + "mae": 0.34535884857177734, + "r2": 0.5475978851318359, + "score": -0.4437945578595596 + }, + "val": { + "rmse": 0.462741360963445, + "mae": 0.35957956314086914, + "r2": 0.5089311599731445, + "score": -0.462741360963445 + }, + "test": { + "rmse": 0.4802330803717953, + "mae": 0.374697744846344, + "r2": 0.48580873012542725, + "score": -0.4802330803717953 + } + }, + "time": "0:14:46.572437", + "diversity": { + "train": { + "mean_centered_corr": -0.02044508419930935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05110829323530197 + }, + "val": { + "mean_centered_corr": -0.020311282947659492, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05272478237748146 + }, + "test": { + "mean_centered_corr": -0.0204484686255455, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053339943289756775 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0ff438b7e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1520160, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.443731832132361, + "mae": 0.3453916907310486, + "r2": 0.5477257966995239, + "score": -0.443731832132361 + }, + "val": { + "rmse": 0.4624384832710735, + "mae": 0.3594527840614319, + "r2": 0.5095738172531128, + "score": -0.4624384832710735 + }, + "test": { + "rmse": 0.47998788853107577, + "mae": 0.37460222840309143, + "r2": 0.4863336682319641, + "score": -0.47998788853107577 + } + }, + "time": "0:14:46.572437", + "diversity": { + "train": { + "mean_centered_corr": -0.02044508419930935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05110829323530197 + }, + "val": { + "mean_centered_corr": -0.020311282947659492, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05272478237748146 + }, + "test": { + "mean_centered_corr": -0.0204484686255455, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053339943289756775 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14, + 3, + 5, + 17, + 22, + 24, + 27, + 10, + 18, + 6, + 5, + 4, + 17, + 3, + 21, + 24, + 22, + 5, + 27 + ], + "head_selection_time": "0:00:00.829666" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..639bab022 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1520160, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44786020595828924, + "mae": 0.348699688911438, + "r2": 0.5392709970474243, + "score": -0.44786020595828924 + }, + "val": { + "rmse": 0.4650147388029769, + "mae": 0.36139675974845886, + "r2": 0.5040942430496216, + "score": -0.4650147388029769 + }, + "test": { + "rmse": 0.4822676099238603, + "mae": 0.37660688161849976, + "r2": 0.4814426898956299, + "score": -0.4822676099238603 + } + }, + "time": "0:14:47.284485", + "diversity": { + "train": { + "mean_centered_corr": -0.020345786586403847, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05121167004108429 + }, + "val": { + "mean_centered_corr": -0.02022434212267399, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052821069955825806 + }, + "test": { + "mean_centered_corr": -0.020434435456991196, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053462423384189606 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 3 + ], + "head_selection_time": "0:00:00.028162" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..cb6260a3f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..b7e742362 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-evaluation/0/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1520160, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44380323735423477, + "mae": 0.3453722298145294, + "r2": 0.5475802421569824, + "score": -0.44380323735423477 + }, + "val": { + "rmse": 0.46274171518445095, + "mae": 0.35957109928131104, + "r2": 0.5089304447174072, + "score": -0.46274171518445095 + }, + "test": { + "rmse": 0.48025462954174786, + "mae": 0.3747461140155792, + "r2": 0.4857625961303711, + "score": -0.48025462954174786 + } + }, + "time": "0:14:47.284485", + "diversity": { + "train": { + "mean_centered_corr": -0.020345786586403847, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05121167004108429 + }, + "val": { + "mean_centered_corr": -0.02022434212267399, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052821069955825806 + }, + "test": { + "mean_centered_corr": -0.020434435456991196, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053462423384189606 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5816ea3be --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1520160, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4437128917866993, + "mae": 0.34536486864089966, + "r2": 0.5477644205093384, + "score": -0.4437128917866993 + }, + "val": { + "rmse": 0.46249997680251603, + "mae": 0.3594898581504822, + "r2": 0.5094433426856995, + "score": -0.46249997680251603 + }, + "test": { + "rmse": 0.47995778958671514, + "mae": 0.37462249398231506, + "r2": 0.4863981008529663, + "score": -0.47995778958671514 + } + }, + "time": "0:14:47.284485", + "diversity": { + "train": { + "mean_centered_corr": -0.020345786586403847, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05121167004108429 + }, + "val": { + "mean_centered_corr": -0.02022434212267399, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052821069955825806 + }, + "test": { + "mean_centered_corr": -0.020434435456991196, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053462423384189606 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 3, + 17, + 27, + 5, + 4, + 14, + 10, + 21, + 22, + 16, + 24, + 31, + 20, + 27, + 17, + 5, + 9, + 10, + 6, + 14 + ], + "head_selection_time": "0:00:00.909476" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9ac54fdad --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44760284038640463, + "mae": 0.3484247922897339, + "r2": 0.5398002862930298, + "score": -0.44760284038640463 + }, + "val": { + "rmse": 0.4651183431272534, + "mae": 0.3614919185638428, + "r2": 0.5038732290267944, + "score": -0.4651183431272534 + }, + "test": { + "rmse": 0.48213882482447107, + "mae": 0.37617027759552, + "r2": 0.48171961307525635, + "score": -0.48213882482447107 + } + }, + "time": "0:23:02.905011", + "diversity": { + "train": { + "mean_centered_corr": -0.020404240116477013, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513404980301857 + }, + "val": { + "mean_centered_corr": -0.02041091024875641, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052927520126104355 + }, + "test": { + "mean_centered_corr": -0.020323792472481728, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05353504791855812 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7 + ], + "head_selection_time": "0:00:00.027779" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..17b62a97a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..0df041ca5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-evaluation/0/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4435648840971451, + "mae": 0.34517785906791687, + "r2": 0.5480660200119019, + "score": -0.4435648840971451 + }, + "val": { + "rmse": 0.4627297681263241, + "mae": 0.35958367586135864, + "r2": 0.5089557766914368, + "score": -0.4627297681263241 + }, + "test": { + "rmse": 0.48025858554781287, + "mae": 0.3747011125087738, + "r2": 0.485754132270813, + "score": -0.48025858554781287 + } + }, + "time": "0:23:02.905011", + "diversity": { + "train": { + "mean_centered_corr": -0.020404240116477013, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513404980301857 + }, + "val": { + "mean_centered_corr": -0.02041091024875641, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052927520126104355 + }, + "test": { + "mean_centered_corr": -0.020323792472481728, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05353504791855812 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8c8acd951 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,177 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44353548828929634, + "mae": 0.3452431261539459, + "r2": 0.5481259822845459, + "score": -0.44353548828929634 + }, + "val": { + "rmse": 0.4624375326912171, + "mae": 0.3594800531864166, + "r2": 0.5095758438110352, + "score": -0.4624375326912171 + }, + "test": { + "rmse": 0.48004674599435365, + "mae": 0.3747219443321228, + "r2": 0.48620766401290894, + "score": -0.48004674599435365 + } + }, + "time": "0:23:02.905011", + "diversity": { + "train": { + "mean_centered_corr": -0.020404240116477013, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513404980301857 + }, + "val": { + "mean_centered_corr": -0.02041091024875641, + "mean_pairwise_disagreement": NaN, + "member_std": 0.052927520126104355 + }, + "test": { + "mean_centered_corr": -0.020323792472481728, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05353504791855812 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7, + 24, + 17, + 14, + 5, + 3, + 16, + 4, + 10, + 6, + 22, + 17, + 27, + 24, + 21, + 5, + 14, + 31, + 17, + 25, + 24, + 28 + ], + "head_selection_time": "0:00:00.985957" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8abb38b47 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4475775717786638, + "mae": 0.3483501672744751, + "r2": 0.5398522615432739, + "score": -0.4475775717786638 + }, + "val": { + "rmse": 0.4649795846661889, + "mae": 0.361278235912323, + "r2": 0.504169225692749, + "score": -0.4649795846661889 + }, + "test": { + "rmse": 0.4827695666979872, + "mae": 0.3767538070678711, + "r2": 0.4803627133369446, + "score": -0.4827695666979872 + } + }, + "time": "0:23:06.357271", + "diversity": { + "train": { + "mean_centered_corr": -0.02037157490849495, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513317808508873 + }, + "val": { + "mean_centered_corr": -0.020479049533605576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05293743312358856 + }, + "test": { + "mean_centered_corr": -0.02046302892267704, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05355745181441307 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7 + ], + "head_selection_time": "0:00:00.028943" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..364d0e3c9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..35053ecb2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4435449287553804, + "mae": 0.3451736271381378, + "r2": 0.5481066703796387, + "score": -0.4435449287553804 + }, + "val": { + "rmse": 0.46272482498106277, + "mae": 0.3595674932003021, + "r2": 0.5089662671089172, + "score": -0.46272482498106277 + }, + "test": { + "rmse": 0.4802920784266626, + "mae": 0.37473270297050476, + "r2": 0.4856823682785034, + "score": -0.4802920784266626 + } + }, + "time": "0:23:06.357271", + "diversity": { + "train": { + "mean_centered_corr": -0.02037157490849495, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513317808508873 + }, + "val": { + "mean_centered_corr": -0.020479049533605576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05293743312358856 + }, + "test": { + "mean_centered_corr": -0.02046302892267704, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05355745181441307 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..298a7858c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44362437525125054, + "mae": 0.3453079164028168, + "r2": 0.5479447841644287, + "score": -0.44362437525125054 + }, + "val": { + "rmse": 0.4624107384246987, + "mae": 0.35940805077552795, + "r2": 0.5096326470375061, + "score": -0.4624107384246987 + }, + "test": { + "rmse": 0.479974942646056, + "mae": 0.37463974952697754, + "r2": 0.48636138439178467, + "score": -0.479974942646056 + } + }, + "time": "0:23:06.357271", + "diversity": { + "train": { + "mean_centered_corr": -0.02037157490849495, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513317808508873 + }, + "val": { + "mean_centered_corr": -0.020479049533605576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05293743312358856 + }, + "test": { + "mean_centered_corr": -0.02046302892267704, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05355745181441307 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7, + 31, + 14, + 17, + 3, + 4, + 5, + 24, + 27, + 22, + 10, + 6, + 14, + 31, + 17, + 3, + 5, + 16, + 24, + 4, + 9, + 27, + 14 + ], + "head_selection_time": "0:00:01.063797" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a9272f18a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44782485316635157, + "mae": 0.348581999540329, + "r2": 0.5393436551094055, + "score": -0.44782485316635157 + }, + "val": { + "rmse": 0.465206517587419, + "mae": 0.36141353845596313, + "r2": 0.5036851167678833, + "score": -0.465206517587419 + }, + "test": { + "rmse": 0.4827162118838719, + "mae": 0.3766613304615021, + "r2": 0.48047757148742676, + "score": -0.4827162118838719 + } + }, + "time": "0:23:03.312853", + "diversity": { + "train": { + "mean_centered_corr": -0.02040313556790352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051265861839056015 + }, + "val": { + "mean_centered_corr": -0.02042475715279579, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05283457785844803 + }, + "test": { + "mean_centered_corr": -0.02050917036831379, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05348861590027809 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7 + ], + "head_selection_time": "0:00:00.027749" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..1f9317e7e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..0aa0dbfa1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-evaluation/0/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44357932932311045, + "mae": 0.34517449140548706, + "r2": 0.5480365753173828, + "score": -0.44357932932311045 + }, + "val": { + "rmse": 0.4627757191386438, + "mae": 0.3595951199531555, + "r2": 0.5088582038879395, + "score": -0.4627757191386438 + }, + "test": { + "rmse": 0.4802501150008962, + "mae": 0.37466320395469666, + "r2": 0.4857722520828247, + "score": -0.4802501150008962 + } + }, + "time": "0:23:03.312853", + "diversity": { + "train": { + "mean_centered_corr": -0.02040313556790352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051265861839056015 + }, + "val": { + "mean_centered_corr": -0.02042475715279579, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05283457785844803 + }, + "test": { + "mean_centered_corr": -0.02050917036831379, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05348861590027809 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c4595c43b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,180 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 1738400, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4435210080380358, + "mae": 0.3451911509037018, + "r2": 0.5481554269790649, + "score": -0.4435210080380358 + }, + "val": { + "rmse": 0.46247985576887213, + "mae": 0.35945838689804077, + "r2": 0.5094860792160034, + "score": -0.46247985576887213 + }, + "test": { + "rmse": 0.47992046986293, + "mae": 0.3745458126068115, + "r2": 0.48647797107696533, + "score": -0.47992046986293 + } + }, + "time": "0:23:03.312853", + "diversity": { + "train": { + "mean_centered_corr": -0.02040313556790352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051265861839056015 + }, + "val": { + "mean_centered_corr": -0.02042475715279579, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05283457785844803 + }, + "test": { + "mean_centered_corr": -0.02050917036831379, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05348861590027809 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7, + 5, + 22, + 27, + 24, + 14, + 17, + 15, + 10, + 3, + 31, + 5, + 16, + 20, + 25, + 4, + 14, + 27, + 17, + 22, + 5, + 10, + 3, + 21, + 24 + ], + "head_selection_time": "0:00:01.189782" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..04178817a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 2174880, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.447447843788818, + "mae": 0.34833618998527527, + "r2": 0.540118932723999, + "score": -0.447447843788818 + }, + "val": { + "rmse": 0.46500539773966587, + "mae": 0.3613583743572235, + "r2": 0.5041141510009766, + "score": -0.46500539773966587 + }, + "test": { + "rmse": 0.4826475534492275, + "mae": 0.3766460120677948, + "r2": 0.48062533140182495, + "score": -0.4826475534492275 + } + }, + "time": "0:40:10.004190", + "diversity": { + "train": { + "mean_centered_corr": -0.020381338894367218, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05130600184202194 + }, + "val": { + "mean_centered_corr": -0.020470399409532547, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05295143276453018 + }, + "test": { + "mean_centered_corr": -0.020449768751859665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053552430123090744 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7 + ], + "head_selection_time": "0:00:00.027926" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..d8590d7b9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..87dcf6175 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-evaluation/0/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 2174880, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4433250077829461, + "mae": 0.34498268365859985, + "r2": 0.5485547184944153, + "score": -0.4433250077829461 + }, + "val": { + "rmse": 0.4627781501968618, + "mae": 0.3595951795578003, + "r2": 0.5088530778884888, + "score": -0.4627781501968618 + }, + "test": { + "rmse": 0.4803483238337491, + "mae": 0.3747539520263672, + "r2": 0.48556190729141235, + "score": -0.4803483238337491 + } + }, + "time": "0:40:10.004190", + "diversity": { + "train": { + "mean_centered_corr": -0.020381338894367218, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05130600184202194 + }, + "val": { + "mean_centered_corr": -0.020470399409532547, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05295143276453018 + }, + "test": { + "mean_centered_corr": -0.020449768751859665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053552430123090744 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d10aadcfd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,176 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 2174880, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4433834556728311, + "mae": 0.34509384632110596, + "r2": 0.5484356880187988, + "score": -0.4433834556728311 + }, + "val": { + "rmse": 0.4625085630139665, + "mae": 0.3594866991043091, + "r2": 0.509425163269043, + "score": -0.4625085630139665 + }, + "test": { + "rmse": 0.4800564927891271, + "mae": 0.3746797442436218, + "r2": 0.48618680238723755, + "score": -0.4800564927891271 + } + }, + "time": "0:40:10.004190", + "diversity": { + "train": { + "mean_centered_corr": -0.020381338894367218, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05130600184202194 + }, + "val": { + "mean_centered_corr": -0.020470399409532547, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05295143276453018 + }, + "test": { + "mean_centered_corr": -0.020449768751859665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053552430123090744 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 7, + 17, + 3, + 5, + 14, + 10, + 6, + 20, + 24, + 25, + 4, + 31, + 27, + 17, + 3, + 5, + 21, + 10, + 0, + 22, + 14 + ], + "head_selection_time": "0:00:00.965959" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a42c8b43a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 2174880, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44626391984830116, + "mae": 0.3475828170776367, + "r2": 0.5425493717193604, + "score": -0.44626391984830116 + }, + "val": { + "rmse": 0.4648571653594056, + "mae": 0.3613499701023102, + "r2": 0.5044302940368652, + "score": -0.4648571653594056 + }, + "test": { + "rmse": 0.4833629710740515, + "mae": 0.37732839584350586, + "r2": 0.47908449172973633, + "score": -0.4833629710740515 + } + }, + "time": "0:40:00.899069", + "diversity": { + "train": { + "mean_centered_corr": -0.020409030839800835, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051384977996349335 + }, + "val": { + "mean_centered_corr": -0.020344730466604233, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05295592173933983 + }, + "test": { + "mean_centered_corr": -0.02032282203435898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053611382842063904 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 17 + ], + "head_selection_time": "0:00:00.029090" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..952aefeab --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..f340087bc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-evaluation/0/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 2174880, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44330914250619985, + "mae": 0.3449688255786896, + "r2": 0.5485870838165283, + "score": -0.44330914250619985 + }, + "val": { + "rmse": 0.4627485258356125, + "mae": 0.3595917522907257, + "r2": 0.5089159607887268, + "score": -0.4627485258356125 + }, + "test": { + "rmse": 0.4803784913673949, + "mae": 0.37476053833961487, + "r2": 0.48549729585647583, + "score": -0.4803784913673949 + } + }, + "time": "0:40:00.899069", + "diversity": { + "train": { + "mean_centered_corr": -0.020409030839800835, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051384977996349335 + }, + "val": { + "mean_centered_corr": -0.020344730466604233, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05295592173933983 + }, + "test": { + "mean_centered_corr": -0.02032282203435898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053611382842063904 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c612469d5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,164 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 2174880, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.44316496767007946, + "mae": 0.34489932656288147, + "r2": 0.5488806366920471, + "score": -0.44316496767007946 + }, + "val": { + "rmse": 0.4624392405112558, + "mae": 0.3594130277633667, + "r2": 0.5095721483230591, + "score": -0.4624392405112558 + }, + "test": { + "rmse": 0.4799761534270821, + "mae": 0.3745183050632477, + "r2": 0.48635876178741455, + "score": -0.4799761534270821 + } + }, + "time": "0:40:00.899069", + "diversity": { + "train": { + "mean_centered_corr": -0.020409030839800835, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051384977996349335 + }, + "val": { + "mean_centered_corr": -0.020344730466604233, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05295592173933983 + }, + "test": { + "mean_centered_corr": -0.02032282203435898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.053611382842063904 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 17, + 3, + 5, + 14, + 27, + 24, + 4, + 25, + 22 + ], + "head_selection_time": "0:00:00.314664" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0a3114588 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,156 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 2174880, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4460647319702049, + "mae": 0.3470715880393982, + "r2": 0.5429576635360718, + "score": -0.4460647319702049 + }, + "val": { + "rmse": 0.46497304704958375, + "mae": 0.36152011156082153, + "r2": 0.5041831731796265, + "score": -0.46497304704958375 + }, + "test": { + "rmse": 0.4824692397862243, + "mae": 0.3767029345035553, + "r2": 0.48100900650024414, + "score": -0.4824692397862243 + } + }, + "time": "0:40:06.616857", + "diversity": { + "train": { + "mean_centered_corr": -0.020358718931674957, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513102225959301 + }, + "val": { + "mean_centered_corr": -0.020433256402611732, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05285926163196564 + }, + "test": { + "mean_centered_corr": -0.020339958369731903, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05363500863313675 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14 + ], + "head_selection_time": "0:00:00.028964" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..80aaa0c1a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/cooking-time" +cat_policy = "ordinal" + +[optimizer] +lr = 0.00012065020494450812 +weight_decay = 0.07000081679295954 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 1 +d_block = 416 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 8 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..dfaebb48b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-evaluation/0/report.json @@ -0,0 +1,152 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 2174880, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.443289545416571, + "mae": 0.3449470102787018, + "r2": 0.5486269593238831, + "score": -0.443289545416571 + }, + "val": { + "rmse": 0.46274024999580365, + "mae": 0.3595466911792755, + "r2": 0.5089335441589355, + "score": -0.46274024999580365 + }, + "test": { + "rmse": 0.48030301468765013, + "mae": 0.374723345041275, + "r2": 0.4856589436531067, + "score": -0.48030301468765013 + } + }, + "time": "0:40:06.616857", + "diversity": { + "train": { + "mean_centered_corr": -0.020358718931674957, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513102225959301 + }, + "val": { + "mean_centered_corr": -0.020433256402611732, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05285926163196564 + }, + "test": { + "mean_centered_corr": -0.020339958369731903, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05363500863313675 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..dfd22beb3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/cooking-time/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,170 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/cooking-time", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.00012065020494450812, + "weight_decay": 0.07000081679295954 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 416, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 8 + } + }, + "n_parameters": 2174880, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 222, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2994, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 416, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 416 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 23088, + "metrics": { + "train": { + "rmse": 0.4431708014670606, + "mae": 0.34492170810699463, + "r2": 0.548868715763092, + "score": -0.4431708014670606 + }, + "val": { + "rmse": 0.46244718340270563, + "mae": 0.3594217300415039, + "r2": 0.5095553398132324, + "score": -0.46244718340270563 + }, + "test": { + "rmse": 0.4799796305248435, + "mae": 0.37459227442741394, + "r2": 0.4863513112068176, + "score": -0.4799796305248435 + } + }, + "time": "0:40:06.616857", + "diversity": { + "train": { + "mean_centered_corr": -0.020358718931674957, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0513102225959301 + }, + "val": { + "mean_centered_corr": -0.020433256402611732, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05285926163196564 + }, + "test": { + "mean_centered_corr": -0.020339958369731903, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05363500863313675 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14, + 3, + 5, + 17, + 10, + 27, + 24, + 4, + 20, + 22, + 5, + 21, + 3, + 25, + 6 + ], + "head_selection_time": "0:00:00.612249" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8043c4861 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-best-head-evaluation/0/report.json @@ -0,0 +1,506 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5163498609634084, + "mae": 0.40700992941856384, + "r2": 0.500271201133728, + "score": -0.5163498609634084 + }, + "val": { + "rmse": 0.557107642501001, + "mae": 0.44366100430488586, + "r2": 0.4147301912307739, + "score": -0.557107642501001 + }, + "test": { + "rmse": 0.5520572896128711, + "mae": 0.4460996389389038, + "r2": 0.43209290504455566, + "score": -0.5520572896128711 + } + }, + "time": "0:30:06.285335", + "diversity": { + "train": { + "mean_centered_corr": -0.019723476842045784, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059378866106271744 + }, + "val": { + "mean_centered_corr": -0.019546858966350555, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06028679013252258 + }, + "test": { + "mean_centered_corr": -0.0195509921759367, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05971246585249901 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.023809" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..48f767b9b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0.toml @@ -0,0 +1,74 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..fe2a56876 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/0/report.json @@ -0,0 +1,502 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5114855052725168, + "mae": 0.4018738567829132, + "r2": 0.5096423625946045, + "score": -0.5114855052725168 + }, + "val": { + "rmse": 0.5560789252392828, + "mae": 0.44038042426109314, + "r2": 0.41688960790634155, + "score": -0.5560789252392828 + }, + "test": { + "rmse": 0.5493995976501785, + "mae": 0.44166451692581177, + "r2": 0.4375477433204651, + "score": -0.5493995976501785 + } + }, + "time": "0:30:06.285335", + "diversity": { + "train": { + "mean_centered_corr": -0.019723476842045784, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059378866106271744 + }, + "val": { + "mean_centered_corr": -0.019546858966350555, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06028679013252258 + }, + "test": { + "mean_centered_corr": -0.0195509921759367, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05971246585249901 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d27b42061 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,510 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5123158821979983, + "mae": 0.40332356095314026, + "r2": 0.5080488920211792, + "score": -0.5123158821979983 + }, + "val": { + "rmse": 0.5558210261849236, + "mae": 0.4417085349559784, + "r2": 0.41743040084838867, + "score": -0.5558210261849236 + }, + "test": { + "rmse": 0.551614874495521, + "mae": 0.4454870820045471, + "r2": 0.433002769947052, + "score": -0.551614874495521 + } + }, + "time": "0:30:06.285335", + "diversity": { + "train": { + "mean_centered_corr": -0.019723476842045784, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059378866106271744 + }, + "val": { + "mean_centered_corr": -0.019546858966350555, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06028679013252258 + }, + "test": { + "mean_centered_corr": -0.0195509921759367, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05971246585249901 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 24, + 30, + 17, + 8, + 12 + ], + "head_selection_time": "0:00:00.148062" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..71c39dbe8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5125628200269988, + "mae": 0.40779775381088257, + "r2": 0.5075745582580566, + "score": -0.5125628200269988 + }, + "val": { + "rmse": 0.5589936165675472, + "mae": 0.4451264441013336, + "r2": 0.41076087951660156, + "score": -0.5589936165675472 + }, + "test": { + "rmse": 0.5554125439499674, + "mae": 0.44996029138565063, + "r2": 0.4251687526702881, + "score": -0.5554125439499674 + } + }, + "time": "0:31:01.859750", + "diversity": { + "train": { + "mean_centered_corr": -0.020016251131892204, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07126544415950775 + }, + "val": { + "mean_centered_corr": -0.020217863842844963, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0740198865532875 + }, + "test": { + "mean_centered_corr": -0.020295117050409317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0730525478720665 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023553" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..0747ce77a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0.toml @@ -0,0 +1,63 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..70c1917f9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/0/report.json @@ -0,0 +1,221 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5021333185484521, + "mae": 0.39498522877693176, + "r2": 0.5274102091789246, + "score": -0.5021333185484521 + }, + "val": { + "rmse": 0.5563682291205534, + "mae": 0.44030144810676575, + "r2": 0.4162827730178833, + "score": -0.5563682291205534 + }, + "test": { + "rmse": 0.5506428213190716, + "mae": 0.44258415699005127, + "r2": 0.43499934673309326, + "score": -0.5506428213190716 + } + }, + "time": "0:31:01.859750", + "diversity": { + "train": { + "mean_centered_corr": -0.020016251131892204, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07126544415950775 + }, + "val": { + "mean_centered_corr": -0.020217863842844963, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0740198865532875 + }, + "test": { + "mean_centered_corr": -0.020295117050409317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0730525478720665 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..96f7f509c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,236 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5032708353541949, + "mae": 0.39703547954559326, + "r2": 0.5252666473388672, + "score": -0.5032708353541949 + }, + "val": { + "rmse": 0.5557239681192615, + "mae": 0.44123438000679016, + "r2": 0.4176338315010071, + "score": -0.5557239681192615 + }, + "test": { + "rmse": 0.5525121073453494, + "mae": 0.44600483775138855, + "r2": 0.43115681409835815, + "score": -0.5525121073453494 + } + }, + "time": "0:31:01.859750", + "diversity": { + "train": { + "mean_centered_corr": -0.020016251131892204, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07126544415950775 + }, + "val": { + "mean_centered_corr": -0.020217863842844963, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0740198865532875 + }, + "test": { + "mean_centered_corr": -0.020295117050409317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0730525478720665 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 13, + 14, + 15, + 17, + 21, + 2, + 29, + 12, + 14, + 22, + 4, + 23 + ], + "head_selection_time": "0:00:00.359604" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ca02d9cd6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-best-head-evaluation/0/report.json @@ -0,0 +1,496 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5186929177280425, + "mae": 0.40856218338012695, + "r2": 0.4957256317138672, + "score": -0.5186929177280425 + }, + "val": { + "rmse": 0.5574872455078781, + "mae": 0.44453755021095276, + "r2": 0.41393232345581055, + "score": -0.5574872455078781 + }, + "test": { + "rmse": 0.5557374285650164, + "mae": 0.4497009515762329, + "r2": 0.4244961142539978, + "score": -0.5557374285650164 + } + }, + "time": "0:16:51.220052", + "diversity": { + "train": { + "mean_centered_corr": -0.019181909039616585, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05811057612299919 + }, + "val": { + "mean_centered_corr": -0.01869083382189274, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05933859571814537 + }, + "test": { + "mean_centered_corr": -0.01882953569293022, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058781228959560394 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.023324" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0.toml new file mode 100644 index 000000000..6c47aea17 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0.toml @@ -0,0 +1,64 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0/report.json new file mode 100644 index 000000000..99f6ba59c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-evaluation/0/report.json @@ -0,0 +1,492 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5143036239160346, + "mae": 0.4036540389060974, + "r2": 0.5042240619659424, + "score": -0.5143036239160346 + }, + "val": { + "rmse": 0.556102345189303, + "mae": 0.44109949469566345, + "r2": 0.41684049367904663, + "score": -0.556102345189303 + }, + "test": { + "rmse": 0.5505710768952337, + "mae": 0.4436887204647064, + "r2": 0.4351465702056885, + "score": -0.5505710768952337 + } + }, + "time": "0:16:51.220052", + "diversity": { + "train": { + "mean_centered_corr": -0.019181909039616585, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05811057612299919 + }, + "val": { + "mean_centered_corr": -0.01869083382189274, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05933859571814537 + }, + "test": { + "mean_centered_corr": -0.01882953569293022, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058781228959560394 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0af0654bc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_esam-greedy-heads-evaluation/0/report.json @@ -0,0 +1,502 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5147739959374277, + "mae": 0.40450477600097656, + "r2": 0.5033168196678162, + "score": -0.5147739959374277 + }, + "val": { + "rmse": 0.5556806886998003, + "mae": 0.4414132237434387, + "r2": 0.41772449016571045, + "score": -0.5556806886998003 + }, + "test": { + "rmse": 0.5513624006625033, + "mae": 0.4449708163738251, + "r2": 0.43352168798446655, + "score": -0.5513624006625033 + } + }, + "time": "0:16:51.220052", + "diversity": { + "train": { + "mean_centered_corr": -0.019181909039616585, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05811057612299919 + }, + "val": { + "mean_centered_corr": -0.01869083382189274, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05933859571814537 + }, + "test": { + "mean_centered_corr": -0.01882953569293022, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058781228959560394 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 24, + 0, + 18, + 19, + 30, + 6, + 28 + ], + "head_selection_time": "0:00:00.202098" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6262c0e23 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,470 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5099830602188714, + "mae": 0.40115422010421753, + "r2": 0.5125188827514648, + "score": -0.5099830602188714 + }, + "val": { + "rmse": 0.5591485795972553, + "mae": 0.4450674057006836, + "r2": 0.41043412685394287, + "score": -0.5591485795972553 + }, + "test": { + "rmse": 0.5570234889363207, + "mae": 0.45008188486099243, + "r2": 0.4218294024467468, + "score": -0.5570234889363207 + } + }, + "time": "0:30:38.784268", + "diversity": { + "train": { + "mean_centered_corr": -0.019464364275336266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06317681819200516 + }, + "val": { + "mean_centered_corr": -0.019761033356189728, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06557753682136536 + }, + "test": { + "mean_centered_corr": -0.019694572314620018, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06528102606534958 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.023362" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0.toml new file mode 100644 index 000000000..79844057e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0.toml @@ -0,0 +1,61 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0/report.json new file mode 100644 index 000000000..74ce93d9d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/0/report.json @@ -0,0 +1,466 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5030298522163873, + "mae": 0.3948743939399719, + "r2": 0.5257211327552795, + "score": -0.5030298522163873 + }, + "val": { + "rmse": 0.5571737580629273, + "mae": 0.4427817463874817, + "r2": 0.4145912528038025, + "score": -0.5571737580629273 + }, + "test": { + "rmse": 0.5527981832493051, + "mae": 0.445404052734375, + "r2": 0.43056756258010864, + "score": -0.5527981832493051 + } + }, + "time": "0:30:38.784268", + "diversity": { + "train": { + "mean_centered_corr": -0.019464364275336266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06317681819200516 + }, + "val": { + "mean_centered_corr": -0.019761033356189728, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06557753682136536 + }, + "test": { + "mean_centered_corr": -0.019694572314620018, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06528102606534958 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8d0a2e3cf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,475 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5034348700541703, + "mae": 0.39474746584892273, + "r2": 0.5249570608139038, + "score": -0.5034348700541703 + }, + "val": { + "rmse": 0.5566172544693042, + "mae": 0.44152215123176575, + "r2": 0.4157600998878479, + "score": -0.5566172544693042 + }, + "test": { + "rmse": 0.5524610510858833, + "mae": 0.4448763132095337, + "r2": 0.43126189708709717, + "score": -0.5524610510858833 + } + }, + "time": "0:30:38.784268", + "diversity": { + "train": { + "mean_centered_corr": -0.019464364275336266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06317681819200516 + }, + "val": { + "mean_centered_corr": -0.019761033356189728, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06557753682136536 + }, + "test": { + "mean_centered_corr": -0.019694572314620018, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06528102606534958 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 29, + 10, + 13, + 15, + 24, + 16 + ], + "head_selection_time": "0:00:00.172357" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a9d3e5d77 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-best-head-evaluation/0/report.json @@ -0,0 +1,495 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5181245330790036, + "mae": 0.4071075916290283, + "r2": 0.49683016538619995, + "score": -0.5181245330790036 + }, + "val": { + "rmse": 0.5572234465535005, + "mae": 0.44068703055381775, + "r2": 0.414486825466156, + "score": -0.5572234465535005 + }, + "test": { + "rmse": 0.5525491087220346, + "mae": 0.44485196471214294, + "r2": 0.43108057975769043, + "score": -0.5525491087220346 + } + }, + "time": "0:14:53.713644", + "diversity": { + "train": { + "mean_centered_corr": -0.018655147403478622, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06323739886283875 + }, + "val": { + "mean_centered_corr": -0.018664149567484856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06284364312887192 + }, + "test": { + "mean_centered_corr": -0.018706250935792923, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06153913214802742 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.023432" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0.toml new file mode 100644 index 000000000..023666cec --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0.toml @@ -0,0 +1,67 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0/report.json new file mode 100644 index 000000000..3cf3fcc90 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/0/report.json @@ -0,0 +1,491 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.514037346731799, + "mae": 0.4048083424568176, + "r2": 0.5047372579574585, + "score": -0.514037346731799 + }, + "val": { + "rmse": 0.5560596579733861, + "mae": 0.44129467010498047, + "r2": 0.41693001985549927, + "score": -0.5560596579733861 + }, + "test": { + "rmse": 0.5513667248174312, + "mae": 0.4448263645172119, + "r2": 0.4335128664970398, + "score": -0.5513667248174312 + } + }, + "time": "0:14:53.713644", + "diversity": { + "train": { + "mean_centered_corr": -0.018655147403478622, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06323739886283875 + }, + "val": { + "mean_centered_corr": -0.018664149567484856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06284364312887192 + }, + "test": { + "mean_centered_corr": -0.018706250935792923, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06153913214802742 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9bc611603 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-greedy-heads-evaluation/0/report.json @@ -0,0 +1,508 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5148022474310279, + "mae": 0.4057745337486267, + "r2": 0.5032622814178467, + "score": -0.5148022474310279 + }, + "val": { + "rmse": 0.5556365745463094, + "mae": 0.44140082597732544, + "r2": 0.41781699657440186, + "score": -0.5556365745463094 + }, + "test": { + "rmse": 0.5515955863733182, + "mae": 0.44562408328056335, + "r2": 0.43304240703582764, + "score": -0.5515955863733182 + } + }, + "time": "0:14:53.713644", + "diversity": { + "train": { + "mean_centered_corr": -0.018655147403478622, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06323739886283875 + }, + "val": { + "mean_centered_corr": -0.018664149567484856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06284364312887192 + }, + "test": { + "mean_centered_corr": -0.018706250935792923, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06153913214802742 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 25, + 24, + 30, + 22, + 19, + 29, + 14, + 23, + 31, + 24, + 12, + 30, + 22, + 6 + ], + "head_selection_time": "0:00:00.434304" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0927cc2e8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-best-head-evaluation/0/report.json @@ -0,0 +1,449 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5140202432442316, + "mae": 0.40386128425598145, + "r2": 0.5047702789306641, + "score": -0.5140202432442316 + }, + "val": { + "rmse": 0.557661572508718, + "mae": 0.44313740730285645, + "r2": 0.4135657548904419, + "score": -0.557661572508718 + }, + "test": { + "rmse": 0.553144673358601, + "mae": 0.44622060656547546, + "r2": 0.42985349893569946, + "score": -0.553144673358601 + } + }, + "time": "0:08:00.261340", + "diversity": { + "train": { + "mean_centered_corr": -0.01827983371913433, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0643538236618042 + }, + "val": { + "mean_centered_corr": -0.018152762204408646, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06669643521308899 + }, + "test": { + "mean_centered_corr": -0.018407996743917465, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06605677306652069 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 30 + ], + "head_selection_time": "0:00:00.023441" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0.toml new file mode 100644 index 000000000..f7bf33592 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0/report.json new file mode 100644 index 000000000..8dc921357 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0/report.json @@ -0,0 +1,445 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5080552327962835, + "mae": 0.3986222445964813, + "r2": 0.5161974430084229, + "score": -0.5080552327962835 + }, + "val": { + "rmse": 0.5562445854609275, + "mae": 0.4416752755641937, + "r2": 0.4165421724319458, + "score": -0.5562445854609275 + }, + "test": { + "rmse": 0.5504166498793033, + "mae": 0.44359084963798523, + "r2": 0.4354633688926697, + "score": -0.5504166498793033 + } + }, + "time": "0:08:00.261340", + "diversity": { + "train": { + "mean_centered_corr": -0.01827983371913433, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0643538236618042 + }, + "val": { + "mean_centered_corr": -0.018152762204408646, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06669643521308899 + }, + "test": { + "mean_centered_corr": -0.018407996743917465, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06605677306652069 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..e627e5168 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,457 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5096923366202544, + "mae": 0.4003257751464844, + "r2": 0.5130745768547058, + "score": -0.5096923366202544 + }, + "val": { + "rmse": 0.5556691040453957, + "mae": 0.4418928027153015, + "r2": 0.4177488088607788, + "score": -0.5556691040453957 + }, + "test": { + "rmse": 0.5521822756750591, + "mae": 0.44639116525650024, + "r2": 0.43183577060699463, + "score": -0.5521822756750591 + } + }, + "time": "0:08:00.261340", + "diversity": { + "train": { + "mean_centered_corr": -0.01827983371913433, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0643538236618042 + }, + "val": { + "mean_centered_corr": -0.018152762204408646, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06669643521308899 + }, + "test": { + "mean_centered_corr": -0.018407996743917465, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06605677306652069 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 30, + 24, + 27, + 15, + 4, + 21, + 24, + 1, + 27 + ], + "head_selection_time": "0:00:00.261765" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f2a4ae64c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-best-head-evaluation/0/report.json @@ -0,0 +1,449 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3003, + "metrics": { + "train": { + "rmse": 0.5091186451729569, + "mae": 0.40107491612434387, + "r2": 0.5141700506210327, + "score": -0.5091186451729569 + }, + "val": { + "rmse": 0.5580564462352832, + "mae": 0.4435662627220154, + "r2": 0.4127349853515625, + "score": -0.5580564462352832 + }, + "test": { + "rmse": 0.5528824139554394, + "mae": 0.4462261497974396, + "r2": 0.4303940534591675, + "score": -0.5528824139554394 + } + }, + "time": "0:09:02.534268", + "diversity": { + "train": { + "mean_centered_corr": -0.017171135172247887, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07136442512273788 + }, + "val": { + "mean_centered_corr": -0.01788639836013317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07444944232702255 + }, + "test": { + "mean_centered_corr": -0.017493220046162605, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07251272350549698 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 27 + ], + "head_selection_time": "0:00:00.023820" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0.toml new file mode 100644 index 000000000..26f1234c9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0/report.json new file mode 100644 index 000000000..8cb92144e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-evaluation/0/report.json @@ -0,0 +1,445 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3003, + "metrics": { + "train": { + "rmse": 0.5000776588212498, + "mae": 0.39380842447280884, + "r2": 0.5312716960906982, + "score": -0.5000776588212498 + }, + "val": { + "rmse": 0.5562691235201772, + "mae": 0.44062381982803345, + "r2": 0.41649067401885986, + "score": -0.5562691235201772 + }, + "test": { + "rmse": 0.5515996385557942, + "mae": 0.4439227879047394, + "r2": 0.4330340623855591, + "score": -0.5515996385557942 + } + }, + "time": "0:09:02.534268", + "diversity": { + "train": { + "mean_centered_corr": -0.017171135172247887, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07136442512273788 + }, + "val": { + "mean_centered_corr": -0.01788639836013317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07444944232702255 + }, + "test": { + "mean_centered_corr": -0.017493220046162605, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07251272350549698 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1b763f269 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json @@ -0,0 +1,469 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3003, + "metrics": { + "train": { + "rmse": 0.5022562202858329, + "mae": 0.39594367146492004, + "r2": 0.5271788239479065, + "score": -0.5022562202858329 + }, + "val": { + "rmse": 0.5559326756199947, + "mae": 0.4413962960243225, + "r2": 0.41719627380371094, + "score": -0.5559326756199947 + }, + "test": { + "rmse": 0.5522105562224877, + "mae": 0.4452051818370819, + "r2": 0.4317775368690491, + "score": -0.5522105562224877 + } + }, + "time": "0:09:02.534268", + "diversity": { + "train": { + "mean_centered_corr": -0.017171135172247887, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07136442512273788 + }, + "val": { + "mean_centered_corr": -0.01788639836013317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07444944232702255 + }, + "test": { + "mean_centered_corr": -0.017493220046162605, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07251272350549698 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 27, + 10, + 7, + 26, + 25, + 17, + 1, + 11, + 28, + 30, + 20, + 9, + 26, + 23, + 19, + 3, + 10, + 12, + 29, + 26, + 24 + ], + "head_selection_time": "0:00:00.749795" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4df5ac783 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-best-head-evaluation/0/report.json @@ -0,0 +1,449 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5165139251966278, + "mae": 0.4059557616710663, + "r2": 0.49995356798171997, + "score": -0.5165139251966278 + }, + "val": { + "rmse": 0.5568247172798395, + "mae": 0.4428383708000183, + "r2": 0.41532450914382935, + "score": -0.5568247172798395 + }, + "test": { + "rmse": 0.5526535729970853, + "mae": 0.4464971721172333, + "r2": 0.43086546659469604, + "score": -0.5526535729970853 + } + }, + "time": "0:07:25.928143", + "diversity": { + "train": { + "mean_centered_corr": -0.018287064507603645, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058509234338998795 + }, + "val": { + "mean_centered_corr": -0.01799921505153179, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05907076969742775 + }, + "test": { + "mean_centered_corr": -0.018038002774119377, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057839713990688324 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.023520" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0.toml new file mode 100644 index 000000000..caf96fb13 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.2 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0/report.json new file mode 100644 index 000000000..6f6b85f0e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-evaluation/0/report.json @@ -0,0 +1,445 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.512394931536762, + "mae": 0.40040844678878784, + "r2": 0.507897138595581, + "score": -0.512394931536762 + }, + "val": { + "rmse": 0.5565664676993906, + "mae": 0.43947651982307434, + "r2": 0.4158666729927063, + "score": -0.5565664676993906 + }, + "test": { + "rmse": 0.5475279860550825, + "mae": 0.438804566860199, + "r2": 0.44137340784072876, + "score": -0.5475279860550825 + } + }, + "time": "0:07:25.928143", + "diversity": { + "train": { + "mean_centered_corr": -0.018287064507603645, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058509234338998795 + }, + "val": { + "mean_centered_corr": -0.01799921505153179, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05907076969742775 + }, + "test": { + "mean_centered_corr": -0.018038002774119377, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057839713990688324 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7c53cfab1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json @@ -0,0 +1,455 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5124534399701177, + "mae": 0.40273600816726685, + "r2": 0.5077847242355347, + "score": -0.5124534399701177 + }, + "val": { + "rmse": 0.5558508909198594, + "mae": 0.4416658878326416, + "r2": 0.41736775636672974, + "score": -0.5558508909198594 + }, + "test": { + "rmse": 0.5511616144232316, + "mae": 0.4449461102485657, + "r2": 0.43393421173095703, + "score": -0.5511616144232316 + } + }, + "time": "0:07:25.928143", + "diversity": { + "train": { + "mean_centered_corr": -0.018287064507603645, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058509234338998795 + }, + "val": { + "mean_centered_corr": -0.01799921505153179, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05907076969742775 + }, + "test": { + "mean_centered_corr": -0.018038002774119377, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057839713990688324 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 25, + 31, + 11, + 17, + 8, + 19, + 23 + ], + "head_selection_time": "0:00:00.201039" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..263f15f32 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,459 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5178361933092778, + "mae": 0.40899282693862915, + "r2": 0.4973900318145752, + "score": -0.5178361933092778 + }, + "val": { + "rmse": 0.5568538056781229, + "mae": 0.4425808787345886, + "r2": 0.4152633547782898, + "score": -0.5568538056781229 + }, + "test": { + "rmse": 0.5520497317854985, + "mae": 0.4457451403141022, + "r2": 0.43210846185684204, + "score": -0.5520497317854985 + } + }, + "time": "0:13:13.542286", + "diversity": { + "train": { + "mean_centered_corr": -0.018485957756638527, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05812399089336395 + }, + "val": { + "mean_centered_corr": -0.01856246404349804, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059649109840393066 + }, + "test": { + "mean_centered_corr": -0.018637534230947495, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05802324041724205 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.023427" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..82ab734b0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0.toml @@ -0,0 +1,54 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..bff979d13 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/0/report.json @@ -0,0 +1,455 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5138792192419979, + "mae": 0.4050156772136688, + "r2": 0.5050419569015503, + "score": -0.5138792192419979 + }, + "val": { + "rmse": 0.5562165100217151, + "mae": 0.44210323691368103, + "r2": 0.4166010618209839, + "score": -0.5562165100217151 + }, + "test": { + "rmse": 0.5500920359593422, + "mae": 0.4439537823200226, + "r2": 0.436129093170166, + "score": -0.5500920359593422 + } + }, + "time": "0:13:13.542286", + "diversity": { + "train": { + "mean_centered_corr": -0.018485957756638527, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05812399089336395 + }, + "val": { + "mean_centered_corr": -0.01856246404349804, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059649109840393066 + }, + "test": { + "mean_centered_corr": -0.018637534230947495, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05802324041724205 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..13d3b92d6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,469 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.51442579053414, + "mae": 0.40507030487060547, + "r2": 0.50398850440979, + "score": -0.51442579053414 + }, + "val": { + "rmse": 0.5558497381815869, + "mae": 0.4416966438293457, + "r2": 0.4173702001571655, + "score": -0.5558497381815869 + }, + "test": { + "rmse": 0.5508600814353861, + "mae": 0.4445980489253998, + "r2": 0.43455344438552856, + "score": -0.5508600814353861 + } + }, + "time": "0:13:13.542286", + "diversity": { + "train": { + "mean_centered_corr": -0.018485957756638527, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05812399089336395 + }, + "val": { + "mean_centered_corr": -0.01856246404349804, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059649109840393066 + }, + "test": { + "mean_centered_corr": -0.018637534230947495, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05802324041724205 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 25, + 31, + 30, + 21, + 26, + 29, + 24, + 12, + 8, + 31, + 18 + ], + "head_selection_time": "0:00:00.326097" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..50d13cf12 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,214 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5044531109629536, + "mae": 0.3947329819202423, + "r2": 0.5230334997177124, + "score": -0.5044531109629536 + }, + "val": { + "rmse": 0.5602888251013873, + "mae": 0.4431866407394409, + "r2": 0.40802711248397827, + "score": -0.5602888251013873 + }, + "test": { + "rmse": 0.5556835311938418, + "mae": 0.44737398624420166, + "r2": 0.42460769414901733, + "score": -0.5556835311938418 + } + }, + "time": "0:15:49.877177", + "diversity": { + "train": { + "mean_centered_corr": -0.020127171650528908, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07589761912822723 + }, + "val": { + "mean_centered_corr": -0.02017100341618061, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07972607016563416 + }, + "test": { + "mean_centered_corr": -0.0202227383852005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07740911841392517 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 7 + ], + "head_selection_time": "0:00:00.023297" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0.toml new file mode 100644 index 000000000..1cbf804d8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0.toml @@ -0,0 +1,56 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0/report.json new file mode 100644 index 000000000..959f64c32 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/0/report.json @@ -0,0 +1,210 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.4988528443427359, + "mae": 0.39271265268325806, + "r2": 0.5335649251937866, + "score": -0.4988528443427359 + }, + "val": { + "rmse": 0.5560703233852616, + "mae": 0.44045791029930115, + "r2": 0.4169076681137085, + "score": -0.5560703233852616 + }, + "test": { + "rmse": 0.5508616503748457, + "mae": 0.443840891122818, + "r2": 0.4345501661300659, + "score": -0.5508616503748457 + } + }, + "time": "0:15:49.877177", + "diversity": { + "train": { + "mean_centered_corr": -0.020127171650528908, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07589761912822723 + }, + "val": { + "mean_centered_corr": -0.02017100341618061, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07972607016563416 + }, + "test": { + "mean_centered_corr": -0.0202227383852005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07740911841392517 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0c954c5a2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,233 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.49913719358770847, + "mae": 0.3932115137577057, + "r2": 0.5330330729484558, + "score": -0.49913719358770847 + }, + "val": { + "rmse": 0.5556538720026133, + "mae": 0.4407852590084076, + "r2": 0.41778069734573364, + "score": -0.5556538720026133 + }, + "test": { + "rmse": 0.5520326723090689, + "mae": 0.4453306496143341, + "r2": 0.43214356899261475, + "score": -0.5520326723090689 + } + }, + "time": "0:15:49.877177", + "diversity": { + "train": { + "mean_centered_corr": -0.020127171650528908, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07589761912822723 + }, + "val": { + "mean_centered_corr": -0.02017100341618061, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07972607016563416 + }, + "test": { + "mean_centered_corr": -0.0202227383852005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07740911841392517 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 7, + 5, + 29, + 15, + 9, + 31, + 19, + 11, + 8, + 21, + 5, + 13, + 28, + 2, + 20, + 12, + 30, + 31, + 22, + 5 + ], + "head_selection_time": "0:00:00.680873" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4b5128852 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-best-head-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5183242601978423, + "mae": 0.41130080819129944, + "r2": 0.49644213914871216, + "score": -0.5183242601978423 + }, + "val": { + "rmse": 0.5577754983463842, + "mae": 0.4452954828739166, + "r2": 0.41332608461380005, + "score": -0.5577754983463842 + }, + "test": { + "rmse": 0.5539871683993082, + "mae": 0.44847947359085083, + "r2": 0.4281154274940491, + "score": -0.5539871683993082 + } + }, + "time": "0:11:17.118588", + "diversity": { + "train": { + "mean_centered_corr": -0.0204029381275177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05960493162274361 + }, + "val": { + "mean_centered_corr": -0.020754853263497353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06137089803814888 + }, + "test": { + "mean_centered_corr": -0.020418910309672356, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06040605902671814 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 10 + ], + "head_selection_time": "0:00:00.023248" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0.toml new file mode 100644 index 000000000..f15292a8e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0/report.json new file mode 100644 index 000000000..bca997f7f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5136864373641319, + "mae": 0.4031018316745758, + "r2": 0.5054132342338562, + "score": -0.5136864373641319 + }, + "val": { + "rmse": 0.5561053998926777, + "mae": 0.44030529260635376, + "r2": 0.41683411598205566, + "score": -0.5561053998926777 + }, + "test": { + "rmse": 0.5493164333767354, + "mae": 0.44202545285224915, + "r2": 0.4377179741859436, + "score": -0.5493164333767354 + } + }, + "time": "0:11:17.118588", + "diversity": { + "train": { + "mean_centered_corr": -0.0204029381275177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05960493162274361 + }, + "val": { + "mean_centered_corr": -0.020754853263497353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06137089803814888 + }, + "test": { + "mean_centered_corr": -0.020418910309672356, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06040605902671814 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..92ef47e51 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,187 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5139899193910727, + "mae": 0.40505242347717285, + "r2": 0.504828691482544, + "score": -0.5139899193910727 + }, + "val": { + "rmse": 0.5555279503030305, + "mae": 0.4421366751194, + "r2": 0.4180445671081543, + "score": -0.5555279503030305 + }, + "test": { + "rmse": 0.5511287648017675, + "mae": 0.4457325041294098, + "r2": 0.4340016841888428, + "score": -0.5511287648017675 + } + }, + "time": "0:11:17.118588", + "diversity": { + "train": { + "mean_centered_corr": -0.0204029381275177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05960493162274361 + }, + "val": { + "mean_centered_corr": -0.020754853263497353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06137089803814888 + }, + "test": { + "mean_centered_corr": -0.020418910309672356, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06040605902671814 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 10, + 21, + 16, + 0, + 15, + 22, + 6, + 11, + 19, + 7 + ], + "head_selection_time": "0:00:00.291162" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1ce08308f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-best-head-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5103791151700686, + "mae": 0.4032345414161682, + "r2": 0.511761486530304, + "score": -0.5103791151700686 + }, + "val": { + "rmse": 0.5594097126680496, + "mae": 0.445120632648468, + "r2": 0.4098833203315735, + "score": -0.5594097126680496 + }, + "test": { + "rmse": 0.5568830263477744, + "mae": 0.44928455352783203, + "r2": 0.42212098836898804, + "score": -0.5568830263477744 + } + }, + "time": "0:12:55.387180", + "diversity": { + "train": { + "mean_centered_corr": -0.020293518900871277, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0689454972743988 + }, + "val": { + "mean_centered_corr": -0.020340915769338608, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07138139009475708 + }, + "test": { + "mean_centered_corr": -0.02027955837547779, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07020118087530136 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 5 + ], + "head_selection_time": "0:00:00.023169" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0.toml new file mode 100644 index 000000000..9e739cfcb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0/report.json new file mode 100644 index 000000000..ee647a6d3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5055408727023979, + "mae": 0.399361252784729, + "r2": 0.5209742784500122, + "score": -0.5055408727023979 + }, + "val": { + "rmse": 0.5559746489354217, + "mae": 0.44288384914398193, + "r2": 0.41710829734802246, + "score": -0.5559746489354217 + }, + "test": { + "rmse": 0.5535077457230605, + "mae": 0.4476969540119171, + "r2": 0.4291048049926758, + "score": -0.5535077457230605 + } + }, + "time": "0:12:55.387180", + "diversity": { + "train": { + "mean_centered_corr": -0.020293518900871277, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0689454972743988 + }, + "val": { + "mean_centered_corr": -0.020340915769338608, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07138139009475708 + }, + "test": { + "mean_centered_corr": -0.02027955837547779, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07020118087530136 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4bf8c5e6d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5057995440474627, + "mae": 0.3991788923740387, + "r2": 0.5204839706420898, + "score": -0.5057995440474627 + }, + "val": { + "rmse": 0.5555843034564808, + "mae": 0.4417176842689514, + "r2": 0.417926549911499, + "score": -0.5555843034564808 + }, + "test": { + "rmse": 0.5528859176747987, + "mae": 0.44678282737731934, + "r2": 0.4303867816925049, + "score": -0.5528859176747987 + } + }, + "time": "0:12:55.387180", + "diversity": { + "train": { + "mean_centered_corr": -0.020293518900871277, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0689454972743988 + }, + "val": { + "mean_centered_corr": -0.020340915769338608, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07138139009475708 + }, + "test": { + "mean_centered_corr": -0.02027955837547779, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07020118087530136 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 5, + 6, + 4, + 12, + 13, + 10, + 18, + 21, + 23, + 24, + 4, + 10, + 19, + 13, + 26, + 9, + 17 + ], + "head_selection_time": "0:00:00.546898" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c0633aa71 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-best-head-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5123514530013489, + "mae": 0.4078504741191864, + "r2": 0.5079806447029114, + "score": -0.5123514530013489 + }, + "val": { + "rmse": 0.5582863288518123, + "mae": 0.4448302686214447, + "r2": 0.41225099563598633, + "score": -0.5582863288518123 + }, + "test": { + "rmse": 0.5560390501031047, + "mae": 0.4517032206058502, + "r2": 0.4238712191581726, + "score": -0.5560390501031047 + } + }, + "time": "0:12:54.891645", + "diversity": { + "train": { + "mean_centered_corr": -0.02027146704494953, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06810352951288223 + }, + "val": { + "mean_centered_corr": -0.02014581859111786, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07032498717308044 + }, + "test": { + "mean_centered_corr": -0.02043480984866619, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0694202110171318 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023468" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0.toml new file mode 100644 index 000000000..4b93a10d8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0/report.json new file mode 100644 index 000000000..616e570ec --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.504551673775848, + "mae": 0.39544814825057983, + "r2": 0.5228471159934998, + "score": -0.504551673775848 + }, + "val": { + "rmse": 0.5556360918188752, + "mae": 0.439733624458313, + "r2": 0.41781800985336304, + "score": -0.5556360918188752 + }, + "test": { + "rmse": 0.5494649592968895, + "mae": 0.4416595995426178, + "r2": 0.43741387128829956, + "score": -0.5494649592968895 + } + }, + "time": "0:12:54.891645", + "diversity": { + "train": { + "mean_centered_corr": -0.02027146704494953, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06810352951288223 + }, + "val": { + "mean_centered_corr": -0.02014581859111786, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07032498717308044 + }, + "test": { + "mean_centered_corr": -0.02043480984866619, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0694202110171318 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7712e89dd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,190 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5047633843312425, + "mae": 0.3963008522987366, + "r2": 0.5224465727806091, + "score": -0.5047633843312425 + }, + "val": { + "rmse": 0.5551513290724005, + "mae": 0.4403177499771118, + "r2": 0.4188333749771118, + "score": -0.5551513290724005 + }, + "test": { + "rmse": 0.5507488105906536, + "mae": 0.44414353370666504, + "r2": 0.43478184938430786, + "score": -0.5507488105906536 + } + }, + "time": "0:12:54.891645", + "diversity": { + "train": { + "mean_centered_corr": -0.02027146704494953, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06810352951288223 + }, + "val": { + "mean_centered_corr": -0.02014581859111786, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07032498717308044 + }, + "test": { + "mean_centered_corr": -0.02043480984866619, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0694202110171318 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 21, + 10, + 6, + 29, + 12, + 17, + 15, + 8, + 22, + 16, + 21, + 4 + ], + "head_selection_time": "0:00:00.401490" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4b82b622b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-best-head-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5210981331659499, + "mae": 0.4159780442714691, + "r2": 0.49103808403015137, + "score": -0.5210981331659499 + }, + "val": { + "rmse": 0.5577866919608585, + "mae": 0.44278469681739807, + "r2": 0.41330254077911377, + "score": -0.5577866919608585 + }, + "test": { + "rmse": 0.5538906498744642, + "mae": 0.44725048542022705, + "r2": 0.4283146262168884, + "score": -0.5538906498744642 + } + }, + "time": "0:11:14.241445", + "diversity": { + "train": { + "mean_centered_corr": -0.02045314945280552, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06041118502616882 + }, + "val": { + "mean_centered_corr": -0.020492304116487503, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06114945560693741 + }, + "test": { + "mean_centered_corr": -0.020352158695459366, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06005072593688965 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.023864" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0.toml new file mode 100644 index 000000000..8a8fdc50a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.01 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0/report.json new file mode 100644 index 000000000..950992b97 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5141867316497518, + "mae": 0.40462085604667664, + "r2": 0.5044494271278381, + "score": -0.5141867316497518 + }, + "val": { + "rmse": 0.555695893192201, + "mae": 0.4408281147480011, + "r2": 0.4176926612854004, + "score": -0.555695893192201 + }, + "test": { + "rmse": 0.5503082950221266, + "mae": 0.44391509890556335, + "r2": 0.4356856346130371, + "score": -0.5503082950221266 + } + }, + "time": "0:11:14.241445", + "diversity": { + "train": { + "mean_centered_corr": -0.02045314945280552, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06041118502616882 + }, + "val": { + "mean_centered_corr": -0.020492304116487503, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06114945560693741 + }, + "test": { + "mean_centered_corr": -0.020352158695459366, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06005072593688965 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b0fcbd994 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5144364211629957, + "mae": 0.40600836277008057, + "r2": 0.5039680004119873, + "score": -0.5144364211629957 + }, + "val": { + "rmse": 0.5553495995574796, + "mae": 0.4413377344608307, + "r2": 0.41841816902160645, + "score": -0.5553495995574796 + }, + "test": { + "rmse": 0.5513616169057922, + "mae": 0.4456392228603363, + "r2": 0.4335232973098755, + "score": -0.5513616169057922 + } + }, + "time": "0:11:14.241445", + "diversity": { + "train": { + "mean_centered_corr": -0.02045314945280552, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06041118502616882 + }, + "val": { + "mean_centered_corr": -0.020492304116487503, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06114945560693741 + }, + "test": { + "mean_centered_corr": -0.020352158695459366, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06005072593688965 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 29, + 18, + 2, + 16, + 15, + 8, + 6, + 22, + 21, + 10, + 11, + 15, + 8, + 19, + 25, + 6, + 16, + 26, + 8, + 15, + 22, + 14 + ], + "head_selection_time": "0:00:00.757838" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..d8b7f4200 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,485 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5194237963617938, + "mae": 0.41060853004455566, + "r2": 0.49430352449417114, + "score": -0.5194237963617938 + }, + "val": { + "rmse": 0.556619663847731, + "mae": 0.44304853677749634, + "r2": 0.415755033493042, + "score": -0.556619663847731 + }, + "test": { + "rmse": 0.55593546321811, + "mae": 0.4501691162586212, + "r2": 0.4240858554840088, + "score": -0.55593546321811 + } + }, + "time": "0:08:29.241452", + "diversity": { + "train": { + "mean_centered_corr": -0.01908317767083645, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06313066184520721 + }, + "val": { + "mean_centered_corr": -0.018972836434841156, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06387995928525925 + }, + "test": { + "mean_centered_corr": -0.019194239750504494, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06254689395427704 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.024039" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..79c621548 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0.toml @@ -0,0 +1,57 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..6010d95f4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-evaluation/0/report.json @@ -0,0 +1,481 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5154945613137585, + "mae": 0.408516526222229, + "r2": 0.5019253492355347, + "score": -0.5154945613137585 + }, + "val": { + "rmse": 0.55593889408891, + "mae": 0.4436722695827484, + "r2": 0.4171832799911499, + "score": -0.55593889408891 + }, + "test": { + "rmse": 0.5537058512136184, + "mae": 0.448915034532547, + "r2": 0.4286960959434509, + "score": -0.5537058512136184 + } + }, + "time": "0:08:29.241452", + "diversity": { + "train": { + "mean_centered_corr": -0.01908317767083645, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06313066184520721 + }, + "val": { + "mean_centered_corr": -0.018972836434841156, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06387995928525925 + }, + "test": { + "mean_centered_corr": -0.019194239750504494, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06254689395427704 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4227cc35c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,494 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5159712120423912, + "mae": 0.4090636968612671, + "r2": 0.5010038614273071, + "score": -0.5159712120423912 + }, + "val": { + "rmse": 0.5555428639260286, + "mae": 0.4416162967681885, + "r2": 0.418013334274292, + "score": -0.5555428639260286 + }, + "test": { + "rmse": 0.5513865344186518, + "mae": 0.44544583559036255, + "r2": 0.4334721565246582, + "score": -0.5513865344186518 + } + }, + "time": "0:08:29.241452", + "diversity": { + "train": { + "mean_centered_corr": -0.01908317767083645, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06313066184520721 + }, + "val": { + "mean_centered_corr": -0.018972836434841156, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06387995928525925 + }, + "test": { + "mean_centered_corr": -0.019194239750504494, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06254689395427704 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 25, + 28, + 20, + 23, + 29, + 14, + 1, + 26, + 12, + 29 + ], + "head_selection_time": "0:00:00.295432" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4d0b207d0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-best-head-evaluation/0/report.json @@ -0,0 +1,204 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7046874761581421, + "min_features_per_member": 142.0, + "mean_features_per_member": 155.03125, + "max_features_per_member": 168.0, + "min_members_per_feature": 16.0, + "mean_members_per_feature": 22.549999237060547, + "max_members_per_feature": 29.0, + "mean_pairwise_jaccard": 0.5446517033602734, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5070207145078325, + "mae": 0.39629772305488586, + "r2": 0.5181657075881958, + "score": -0.5070207145078325 + }, + "val": { + "rmse": 0.5595885535124743, + "mae": 0.4446312487125397, + "r2": 0.4095059037208557, + "score": -0.5595885535124743 + }, + "test": { + "rmse": 0.5534224254724212, + "mae": 0.44561949372291565, + "r2": 0.42928075790405273, + "score": -0.5534224254724212 + } + }, + "time": "0:09:13.835446", + "diversity": { + "train": { + "mean_centered_corr": 0.022733226418495178, + "mean_pairwise_disagreement": NaN, + "member_std": 0.12898561358451843 + }, + "val": { + "mean_centered_corr": 0.03887920081615448, + "mean_pairwise_disagreement": NaN, + "member_std": 0.14495275914669037 + }, + "test": { + "mean_centered_corr": 0.04169675335288048, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1456397920846939 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.023598" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0.toml new file mode 100644 index 000000000..040e3d71f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.7 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0/report.json new file mode 100644 index 000000000..87e75b5e0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7046874761581421, + "min_features_per_member": 142.0, + "mean_features_per_member": 155.03125, + "max_features_per_member": 168.0, + "min_members_per_feature": 16.0, + "mean_members_per_feature": 22.549999237060547, + "max_members_per_feature": 29.0, + "mean_pairwise_jaccard": 0.5446517033602734, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5028727675143494, + "mae": 0.39573734998703003, + "r2": 0.5260173082351685, + "score": -0.5028727675143494 + }, + "val": { + "rmse": 0.5576335149669441, + "mae": 0.4440232217311859, + "r2": 0.41362476348876953, + "score": -0.5576335149669441 + }, + "test": { + "rmse": 0.5507854163911843, + "mae": 0.44484779238700867, + "r2": 0.4347066879272461, + "score": -0.5507854163911843 + } + }, + "time": "0:09:13.835446", + "diversity": { + "train": { + "mean_centered_corr": 0.022733226418495178, + "mean_pairwise_disagreement": NaN, + "member_std": 0.12898561358451843 + }, + "val": { + "mean_centered_corr": 0.03887920081615448, + "mean_pairwise_disagreement": NaN, + "member_std": 0.14495275914669037 + }, + "test": { + "mean_centered_corr": 0.04169675335288048, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1456397920846939 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..22735cb5d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.7-greedy-heads-evaluation/0/report.json @@ -0,0 +1,211 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7046874761581421, + "min_features_per_member": 142.0, + "mean_features_per_member": 155.03125, + "max_features_per_member": 168.0, + "min_members_per_feature": 16.0, + "mean_members_per_feature": 22.549999237060547, + "max_members_per_feature": 29.0, + "mean_pairwise_jaccard": 0.5446517033602734, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.503753402357704, + "mae": 0.3957558870315552, + "r2": 0.5243557691574097, + "score": -0.503753402357704 + }, + "val": { + "rmse": 0.5551008108444078, + "mae": 0.43975356221199036, + "r2": 0.41893917322158813, + "score": -0.5551008108444078 + }, + "test": { + "rmse": 0.5501919833980724, + "mae": 0.44342172145843506, + "r2": 0.4359241724014282, + "score": -0.5501919833980724 + } + }, + "time": "0:09:13.835446", + "diversity": { + "train": { + "mean_centered_corr": 0.022733226418495178, + "mean_pairwise_disagreement": NaN, + "member_std": 0.12898561358451843 + }, + "val": { + "mean_centered_corr": 0.03887920081615448, + "mean_pairwise_disagreement": NaN, + "member_std": 0.14495275914669037 + }, + "test": { + "mean_centered_corr": 0.04169675335288048, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1456397920846939 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 8, + 11, + 4, + 16, + 2, + 7, + 23, + 18 + ], + "head_selection_time": "0:00:00.227173" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-best-head-evaluation/0/report.json new file mode 100644 index 000000000..26f8a4eed --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-best-head-evaluation/0/report.json @@ -0,0 +1,204 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8049715757369995, + "min_features_per_member": 164.0, + "mean_features_per_member": 177.09375, + "max_features_per_member": 189.0, + "min_members_per_feature": 20.0, + "mean_members_per_feature": 25.759090423583984, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6745446995892237, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5119150673318991, + "mae": 0.40731897950172424, + "r2": 0.5088183879852295, + "score": -0.5119150673318991 + }, + "val": { + "rmse": 0.5594878077640214, + "mae": 0.44474470615386963, + "r2": 0.40971851348876953, + "score": -0.5594878077640214 + }, + "test": { + "rmse": 0.5558040556519808, + "mae": 0.4507056474685669, + "r2": 0.4243581295013428, + "score": -0.5558040556519808 + } + }, + "time": "0:09:15.530137", + "diversity": { + "train": { + "mean_centered_corr": 0.008296106941998005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10944075882434845 + }, + "val": { + "mean_centered_corr": 0.0186319500207901, + "mean_pairwise_disagreement": NaN, + "member_std": 0.12241803109645844 + }, + "test": { + "mean_centered_corr": 0.021373353898525238, + "mean_pairwise_disagreement": NaN, + "member_std": 0.12239326536655426 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.023292" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0.toml new file mode 100644 index 000000000..4c7765d8f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.8 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0/report.json new file mode 100644 index 000000000..4c0521908 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-evaluation/0/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8049715757369995, + "min_features_per_member": 164.0, + "mean_features_per_member": 177.09375, + "max_features_per_member": 189.0, + "min_members_per_feature": 20.0, + "mean_members_per_feature": 25.759090423583984, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6745446995892237, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.4994902993063076, + "mae": 0.3910210430622101, + "r2": 0.5323721170425415, + "score": -0.4994902993063076 + }, + "val": { + "rmse": 0.5567305910455574, + "mae": 0.4403575658798218, + "r2": 0.41552215814590454, + "score": -0.5567305910455574 + }, + "test": { + "rmse": 0.5489749915571137, + "mae": 0.4408988654613495, + "r2": 0.43841683864593506, + "score": -0.5489749915571137 + } + }, + "time": "0:09:15.530137", + "diversity": { + "train": { + "mean_centered_corr": 0.008296106941998005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10944075882434845 + }, + "val": { + "mean_centered_corr": 0.0186319500207901, + "mean_pairwise_disagreement": NaN, + "member_std": 0.12241803109645844 + }, + "test": { + "mean_centered_corr": 0.021373353898525238, + "mean_pairwise_disagreement": NaN, + "member_std": 0.12239326536655426 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ad4c50f24 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.8-greedy-heads-evaluation/0/report.json @@ -0,0 +1,220 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8049715757369995, + "min_features_per_member": 164.0, + "mean_features_per_member": 177.09375, + "max_features_per_member": 189.0, + "min_members_per_feature": 20.0, + "mean_members_per_feature": 25.759090423583984, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6745446995892237, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5003759937395629, + "mae": 0.39189252257347107, + "r2": 0.5307123064994812, + "score": -0.5003759937395629 + }, + "val": { + "rmse": 0.5554990875501031, + "mae": 0.43922579288482666, + "r2": 0.4181050658226013, + "score": -0.5554990875501031 + }, + "test": { + "rmse": 0.5493161621093208, + "mae": 0.4414233863353729, + "r2": 0.43771857023239136, + "score": -0.5493161621093208 + } + }, + "time": "0:09:15.530137", + "diversity": { + "train": { + "mean_centered_corr": 0.008296106941998005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10944075882434845 + }, + "val": { + "mean_centered_corr": 0.0186319500207901, + "mean_pairwise_disagreement": NaN, + "member_std": 0.12241803109645844 + }, + "test": { + "mean_centered_corr": 0.021373353898525238, + "mean_pairwise_disagreement": NaN, + "member_std": 0.12239326536655426 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4, + 22, + 27, + 15, + 2, + 7, + 23, + 21, + 8, + 10, + 18, + 25, + 27, + 28, + 12, + 2, + 7 + ], + "head_selection_time": "0:00:00.551471" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e71b44bed --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-best-head-evaluation/0/report.json @@ -0,0 +1,204 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5157195206907822, + "mae": 0.4108550548553467, + "r2": 0.501490592956543, + "score": -0.5157195206907822 + }, + "val": { + "rmse": 0.5590040394051904, + "mae": 0.4459855556488037, + "r2": 0.41073888540267944, + "score": -0.5590040394051904 + }, + "test": { + "rmse": 0.5557457406296488, + "mae": 0.45139390230178833, + "r2": 0.42447882890701294, + "score": -0.5557457406296488 + } + }, + "time": "0:08:30.539944", + "diversity": { + "train": { + "mean_centered_corr": -0.02001963183283806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07454843819141388 + }, + "val": { + "mean_centered_corr": -0.020045898854732513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650969177484512 + }, + "test": { + "mean_centered_corr": -0.02007085457444191, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0762816071510315 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023644" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0.toml new file mode 100644 index 000000000..8328925cb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0/report.json new file mode 100644 index 000000000..dea53c836 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-evaluation/0/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.505005369677181, + "mae": 0.3960992991924286, + "r2": 0.521988570690155, + "score": -0.505005369677181 + }, + "val": { + "rmse": 0.5560026829864069, + "mae": 0.4405945837497711, + "r2": 0.4170495271682739, + "score": -0.5560026829864069 + }, + "test": { + "rmse": 0.5501201530906976, + "mae": 0.4423939883708954, + "r2": 0.43607139587402344, + "score": -0.5501201530906976 + } + }, + "time": "0:08:30.539944", + "diversity": { + "train": { + "mean_centered_corr": -0.02001963183283806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07454843819141388 + }, + "val": { + "mean_centered_corr": -0.020045898854732513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650969177484512 + }, + "test": { + "mean_centered_corr": -0.02007085457444191, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0762816071510315 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..aae596674 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.9-greedy-heads-evaluation/0/report.json @@ -0,0 +1,217 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.505321820939252, + "mae": 0.3968241810798645, + "r2": 0.5213893055915833, + "score": -0.505321820939252 + }, + "val": { + "rmse": 0.5555016895512279, + "mae": 0.4406256079673767, + "r2": 0.418099582195282, + "score": -0.5555016895512279 + }, + "test": { + "rmse": 0.5503428181880856, + "mae": 0.44352859258651733, + "r2": 0.43561482429504395, + "score": -0.5503428181880856 + } + }, + "time": "0:08:30.539944", + "diversity": { + "train": { + "mean_centered_corr": -0.02001963183283806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07454843819141388 + }, + "val": { + "mean_centered_corr": -0.020045898854732513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650969177484512 + }, + "test": { + "mean_centered_corr": -0.02007085457444191, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0762816071510315 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 25, + 26, + 15, + 23, + 17, + 6, + 4, + 24, + 21, + 12, + 6, + 10 + ], + "head_selection_time": "0:00:00.431663" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-best-head-evaluation/0/report.json new file mode 100644 index 000000000..09d10242a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-best-head-evaluation/0/report.json @@ -0,0 +1,204 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9481534361839294, + "min_features_per_member": 201.0, + "mean_features_per_member": 208.59375, + "max_features_per_member": 216.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.340909957885742, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9014792827014309, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.51368986033004, + "mae": 0.4090161621570587, + "r2": 0.5054066777229309, + "score": -0.51368986033004 + }, + "val": { + "rmse": 0.5584568843028204, + "mae": 0.4441465139389038, + "r2": 0.4118918776512146, + "score": -0.5584568843028204 + }, + "test": { + "rmse": 0.5549022634897627, + "mae": 0.449065625667572, + "r2": 0.4262245297431946, + "score": -0.5549022634897627 + } + }, + "time": "0:08:30.501515", + "diversity": { + "train": { + "mean_centered_corr": -0.020021812990307808, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07152318209409714 + }, + "val": { + "mean_centered_corr": -0.020280156284570694, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0745302140712738 + }, + "test": { + "mean_centered_corr": -0.019991939887404442, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07362524420022964 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023937" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0.toml new file mode 100644 index 000000000..f3a888a1c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0/report.json new file mode 100644 index 000000000..f258aa6ac --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-evaluation/0/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9481534361839294, + "min_features_per_member": 201.0, + "mean_features_per_member": 208.59375, + "max_features_per_member": 216.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.340909957885742, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9014792827014309, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5051799615450683, + "mae": 0.3969549536705017, + "r2": 0.5216580033302307, + "score": -0.5051799615450683 + }, + "val": { + "rmse": 0.5557158166151223, + "mae": 0.4399119019508362, + "r2": 0.41765087842941284, + "score": -0.5557158166151223 + }, + "test": { + "rmse": 0.5496765041498511, + "mae": 0.4418066442012787, + "r2": 0.43698060512542725, + "score": -0.5496765041498511 + } + }, + "time": "0:08:30.501515", + "diversity": { + "train": { + "mean_centered_corr": -0.020021812990307808, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07152318209409714 + }, + "val": { + "mean_centered_corr": -0.020280156284570694, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0745302140712738 + }, + "test": { + "mean_centered_corr": -0.019991939887404442, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07362524420022964 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1ca17b257 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/mfb_keep0.95-greedy-heads-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9481534361839294, + "min_features_per_member": 201.0, + "mean_features_per_member": 208.59375, + "max_features_per_member": 216.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.340909957885742, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9014792827014309, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.506557145369867, + "mae": 0.3995445668697357, + "r2": 0.519046425819397, + "score": -0.506557145369867 + }, + "val": { + "rmse": 0.5552807442811304, + "mae": 0.4407994747161865, + "r2": 0.4185624122619629, + "score": -0.5552807442811304 + }, + "test": { + "rmse": 0.5512821544083845, + "mae": 0.4446823000907898, + "r2": 0.43368661403656006, + "score": -0.5512821544083845 + } + }, + "time": "0:08:30.501515", + "diversity": { + "train": { + "mean_centered_corr": -0.020021812990307808, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07152318209409714 + }, + "val": { + "mean_centered_corr": -0.020280156284570694, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0745302140712738 + }, + "test": { + "mean_centered_corr": -0.019991939887404442, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07362524420022964 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 23, + 18, + 17, + 21, + 12, + 6, + 8, + 4, + 17 + ], + "head_selection_time": "0:00:00.291365" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3314112db --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,460 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5174210795978582, + "mae": 0.40967971086502075, + "r2": 0.4981955885887146, + "score": -0.5174210795978582 + }, + "val": { + "rmse": 0.5567133805762534, + "mae": 0.4420411288738251, + "r2": 0.4155582785606384, + "score": -0.5567133805762534 + }, + "test": { + "rmse": 0.5518657747493417, + "mae": 0.44505882263183594, + "r2": 0.432486891746521, + "score": -0.5518657747493417 + } + }, + "time": "0:16:08.700759", + "diversity": { + "train": { + "mean_centered_corr": -0.019840920343995094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.054146915674209595 + }, + "val": { + "mean_centered_corr": -0.01915006712079048, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0556391216814518 + }, + "test": { + "mean_centered_corr": -0.019136715680360794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05420564487576485 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.023179" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..28ed0aa45 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..bc6b9c016 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-evaluation/0/report.json @@ -0,0 +1,456 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5117054420662647, + "mae": 0.4039783775806427, + "r2": 0.5092206001281738, + "score": -0.5117054420662647 + }, + "val": { + "rmse": 0.556067616855386, + "mae": 0.4418751299381256, + "r2": 0.41691333055496216, + "score": -0.556067616855386 + }, + "test": { + "rmse": 0.5512732344334902, + "mae": 0.44450679421424866, + "r2": 0.43370485305786133, + "score": -0.5512732344334902 + } + }, + "time": "0:16:08.700759", + "diversity": { + "train": { + "mean_centered_corr": -0.019840920343995094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.054146915674209595 + }, + "val": { + "mean_centered_corr": -0.01915006712079048, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0556391216814518 + }, + "test": { + "mean_centered_corr": -0.019136715680360794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05420564487576485 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7d9df2857 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,468 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5125765417673701, + "mae": 0.40452396869659424, + "r2": 0.5075482130050659, + "score": -0.5125765417673701 + }, + "val": { + "rmse": 0.55586820851409, + "mae": 0.44162702560424805, + "r2": 0.4173314571380615, + "score": -0.55586820851409 + }, + "test": { + "rmse": 0.5514142072129953, + "mae": 0.4448014497756958, + "r2": 0.4334152340888977, + "score": -0.5514142072129953 + } + }, + "time": "0:16:08.700759", + "diversity": { + "train": { + "mean_centered_corr": -0.019840920343995094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.054146915674209595 + }, + "val": { + "mean_centered_corr": -0.01915006712079048, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0556391216814518 + }, + "test": { + "mean_centered_corr": -0.019136715680360794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05420564487576485 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 29, + 31, + 15, + 1, + 27, + 28, + 18, + 8, + 25 + ], + "head_selection_time": "0:00:00.261867" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-best-head-evaluation/0/report.json new file mode 100644 index 000000000..97ec96d27 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-best-head-evaluation/0/report.json @@ -0,0 +1,189 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5197141511819273, + "mae": 0.41487401723861694, + "r2": 0.49373799562454224, + "score": -0.5197141511819273 + }, + "val": { + "rmse": 0.5572907783524989, + "mae": 0.4426497519016266, + "r2": 0.41434532403945923, + "score": -0.5572907783524989 + }, + "test": { + "rmse": 0.5545303498210034, + "mae": 0.44824421405792236, + "r2": 0.42699337005615234, + "score": -0.5545303498210034 + } + }, + "time": "0:25:14.112066", + "diversity": { + "train": { + "mean_centered_corr": -0.020538095384836197, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056509166955947876 + }, + "val": { + "mean_centered_corr": -0.020886661484837532, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058363690972328186 + }, + "test": { + "mean_centered_corr": -0.020847400650382042, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057258859276771545 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.023366" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0.toml new file mode 100644 index 000000000..ce432034e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0/report.json new file mode 100644 index 000000000..f51c2c83d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0/report.json @@ -0,0 +1,185 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5146695445761407, + "mae": 0.40757593512535095, + "r2": 0.5035183429718018, + "score": -0.5146695445761407 + }, + "val": { + "rmse": 0.5559417888696784, + "mae": 0.44435763359069824, + "r2": 0.4171772003173828, + "score": -0.5559417888696784 + }, + "test": { + "rmse": 0.5541331785404355, + "mae": 0.44994547963142395, + "r2": 0.42781394720077515, + "score": -0.5541331785404355 + } + }, + "time": "0:25:14.112066", + "diversity": { + "train": { + "mean_centered_corr": -0.020538095384836197, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056509166955947876 + }, + "val": { + "mean_centered_corr": -0.020886661484837532, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058363690972328186 + }, + "test": { + "mean_centered_corr": -0.020847400650382042, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057258859276771545 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5449bd7de --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-greedy-heads-evaluation/0/report.json @@ -0,0 +1,196 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5146023695303711, + "mae": 0.4073488712310791, + "r2": 0.5036479234695435, + "score": -0.5146023695303711 + }, + "val": { + "rmse": 0.5552988041877751, + "mae": 0.44173434376716614, + "r2": 0.4185245633125305, + "score": -0.5552988041877751 + }, + "test": { + "rmse": 0.5525187688548042, + "mae": 0.44692304730415344, + "r2": 0.43114304542541504, + "score": -0.5525187688548042 + } + }, + "time": "0:25:14.112066", + "diversity": { + "train": { + "mean_centered_corr": -0.020538095384836197, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056509166955947876 + }, + "val": { + "mean_centered_corr": -0.020886661484837532, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058363690972328186 + }, + "test": { + "mean_centered_corr": -0.020847400650382042, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057258859276771545 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 29, + 16, + 11, + 10, + 26, + 8, + 15, + 22 + ], + "head_selection_time": "0:00:00.230227" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..128269a30 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,215 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5163692824712737, + "mae": 0.40596020221710205, + "r2": 0.5002335906028748, + "score": -0.5163692824712737 + }, + "val": { + "rmse": 0.5579516581852805, + "mae": 0.44197753071784973, + "r2": 0.41295546293258667, + "score": -0.5579516581852805 + }, + "test": { + "rmse": 0.5533480523154869, + "mae": 0.44432690739631653, + "r2": 0.4294341802597046, + "score": -0.5533480523154869 + } + }, + "time": "0:16:07.995080", + "diversity": { + "train": { + "mean_centered_corr": -0.02017831802368164, + "mean_pairwise_disagreement": NaN, + "member_std": 0.062000252306461334 + }, + "val": { + "mean_centered_corr": -0.02033226191997528, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06415332108736038 + }, + "test": { + "mean_centered_corr": -0.02038513869047165, + "mean_pairwise_disagreement": NaN, + "member_std": 0.063583143055439 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 14 + ], + "head_selection_time": "0:00:00.023639" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0.toml new file mode 100644 index 000000000..cfe3ed79e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0.toml @@ -0,0 +1,53 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0/report.json new file mode 100644 index 000000000..b61d36bce --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-evaluation/0/report.json @@ -0,0 +1,211 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.513211874722488, + "mae": 0.4028632938861847, + "r2": 0.5063266754150391, + "score": -0.513211874722488 + }, + "val": { + "rmse": 0.5559017430643394, + "mae": 0.4410461187362671, + "r2": 0.41726118326187134, + "score": -0.5559017430643394 + }, + "test": { + "rmse": 0.5492186599232151, + "mae": 0.44233980774879456, + "r2": 0.43791818618774414, + "score": -0.5492186599232151 + } + }, + "time": "0:16:07.995080", + "diversity": { + "train": { + "mean_centered_corr": -0.02017831802368164, + "mean_pairwise_disagreement": NaN, + "member_std": 0.062000252306461334 + }, + "val": { + "mean_centered_corr": -0.02033226191997528, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06415332108736038 + }, + "test": { + "mean_centered_corr": -0.02038513869047165, + "mean_pairwise_disagreement": NaN, + "member_std": 0.063583143055439 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..355bf260f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,221 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9009943008422852, + "min_features_per_member": 188.0, + "mean_features_per_member": 198.21875, + "max_features_per_member": 206.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.831817626953125, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8196317439934785, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5135637466863064, + "mae": 0.40484777092933655, + "r2": 0.5056494474411011, + "score": -0.5135637466863064 + }, + "val": { + "rmse": 0.5554524372350571, + "mae": 0.4416925311088562, + "r2": 0.4182027578353882, + "score": -0.5554524372350571 + }, + "test": { + "rmse": 0.5505269863779684, + "mae": 0.4447595179080963, + "r2": 0.4352370500564575, + "score": -0.5505269863779684 + } + }, + "time": "0:16:07.995080", + "diversity": { + "train": { + "mean_centered_corr": -0.02017831802368164, + "mean_pairwise_disagreement": NaN, + "member_std": 0.062000252306461334 + }, + "val": { + "mean_centered_corr": -0.02033226191997528, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06415332108736038 + }, + "test": { + "mean_centered_corr": -0.02038513869047165, + "mean_pairwise_disagreement": NaN, + "member_std": 0.063583143055439 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 14, + 6, + 11, + 8, + 16, + 15, + 22 + ], + "head_selection_time": "0:00:00.201384" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..472f66269 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5172845259639738, + "mae": 0.4081471264362335, + "r2": 0.49846041202545166, + "score": -0.5172845259639738 + }, + "val": { + "rmse": 0.5574308976082757, + "mae": 0.44358816742897034, + "r2": 0.41405075788497925, + "score": -0.5574308976082757 + }, + "test": { + "rmse": 0.5550464489547554, + "mae": 0.449802041053772, + "r2": 0.4259263277053833, + "score": -0.5550464489547554 + } + }, + "time": "0:09:32.457201", + "diversity": { + "train": { + "mean_centered_corr": -0.020586222410202026, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057862311601638794 + }, + "val": { + "mean_centered_corr": -0.020692767575383186, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05938911437988281 + }, + "test": { + "mean_centered_corr": -0.02092323824763298, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058085475116968155 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.023792" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..50e88d179 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..ee601d869 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5137669870487251, + "mae": 0.4040716290473938, + "r2": 0.5052582025527954, + "score": -0.5137669870487251 + }, + "val": { + "rmse": 0.5558827645265865, + "mae": 0.4413273334503174, + "r2": 0.4173009395599365, + "score": -0.5558827645265865 + }, + "test": { + "rmse": 0.5504866820118107, + "mae": 0.44446638226509094, + "r2": 0.435319721698761, + "score": -0.5504866820118107 + } + }, + "time": "0:09:32.457201", + "diversity": { + "train": { + "mean_centered_corr": -0.020586222410202026, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057862311601638794 + }, + "val": { + "mean_centered_corr": -0.020692767575383186, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05938911437988281 + }, + "test": { + "mean_centered_corr": -0.02092323824763298, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058085475116968155 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c0804ba27 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5145618575810417, + "mae": 0.40636324882507324, + "r2": 0.503726065158844, + "score": -0.5145618575810417 + }, + "val": { + "rmse": 0.5553755991930566, + "mae": 0.44143426418304443, + "r2": 0.41836369037628174, + "score": -0.5553755991930566 + }, + "test": { + "rmse": 0.5516112816611277, + "mae": 0.4461708664894104, + "r2": 0.43301016092300415, + "score": -0.5516112816611277 + } + }, + "time": "0:09:32.457201", + "diversity": { + "train": { + "mean_centered_corr": -0.020586222410202026, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057862311601638794 + }, + "val": { + "mean_centered_corr": -0.020692767575383186, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05938911437988281 + }, + "test": { + "mean_centered_corr": -0.02092323824763298, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058085475116968155 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 21, + 15, + 31, + 16, + 29, + 10, + 13, + 11, + 26, + 14 + ], + "head_selection_time": "0:00:00.289768" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c496cfa72 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5169435461643365, + "mae": 0.4078221619129181, + "r2": 0.49912136793136597, + "score": -0.5169435461643365 + }, + "val": { + "rmse": 0.5574634025946379, + "mae": 0.44402608275413513, + "r2": 0.41398245096206665, + "score": -0.5574634025946379 + }, + "test": { + "rmse": 0.5547066537478277, + "mae": 0.450179785490036, + "r2": 0.4266290068626404, + "score": -0.5547066537478277 + } + }, + "time": "0:09:33.111358", + "diversity": { + "train": { + "mean_centered_corr": -0.020444847643375397, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059093035757541656 + }, + "val": { + "mean_centered_corr": -0.020575184375047684, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06087604537606239 + }, + "test": { + "mean_centered_corr": -0.020676201209425926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060276810079813004 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 18 + ], + "head_selection_time": "0:00:00.023373" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..6d6f2bc2b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..03a93045c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5137012893899691, + "mae": 0.40439915657043457, + "r2": 0.5053846836090088, + "score": -0.5137012893899691 + }, + "val": { + "rmse": 0.5556336245387727, + "mae": 0.4420306384563446, + "r2": 0.4178231954574585, + "score": -0.5556336245387727 + }, + "test": { + "rmse": 0.5518591323593222, + "mae": 0.4463585913181305, + "r2": 0.43250054121017456, + "score": -0.5518591323593222 + } + }, + "time": "0:09:33.111358", + "diversity": { + "train": { + "mean_centered_corr": -0.020444847643375397, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059093035757541656 + }, + "val": { + "mean_centered_corr": -0.020575184375047684, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06087604537606239 + }, + "test": { + "mean_centered_corr": -0.020676201209425926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060276810079813004 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..882c6c418 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.514097726188904, + "mae": 0.4052426517009735, + "r2": 0.5046209692955017, + "score": -0.514097726188904 + }, + "val": { + "rmse": 0.5552720495549479, + "mae": 0.441690057516098, + "r2": 0.4185805916786194, + "score": -0.5552720495549479 + }, + "test": { + "rmse": 0.5524877260752137, + "mae": 0.447174072265625, + "r2": 0.43120694160461426, + "score": -0.5524877260752137 + } + }, + "time": "0:09:33.111358", + "diversity": { + "train": { + "mean_centered_corr": -0.020444847643375397, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059093035757541656 + }, + "val": { + "mean_centered_corr": -0.020575184375047684, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06087604537606239 + }, + "test": { + "mean_centered_corr": -0.020676201209425926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060276810079813004 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 18, + 10, + 11, + 26, + 21, + 14, + 31, + 19, + 11, + 16, + 26, + 10, + 11, + 15, + 21, + 29 + ], + "head_selection_time": "0:00:00.514504" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..afb9cebaa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5198625067814638, + "mae": 0.4135857820510864, + "r2": 0.4934489130973816, + "score": -0.5198625067814638 + }, + "val": { + "rmse": 0.5574473374558679, + "mae": 0.44392451643943787, + "r2": 0.4140162467956543, + "score": -0.5574473374558679 + }, + "test": { + "rmse": 0.5570882235759158, + "mae": 0.45243602991104126, + "r2": 0.4216949939727783, + "score": -0.5570882235759158 + } + }, + "time": "0:09:32.630353", + "diversity": { + "train": { + "mean_centered_corr": -0.02049259841442108, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05744750052690506 + }, + "val": { + "mean_centered_corr": -0.020656166598200798, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05917323753237724 + }, + "test": { + "mean_centered_corr": -0.020809270441532135, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05833838880062103 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 14 + ], + "head_selection_time": "0:00:00.023420" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..0d48e7096 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..bc695d15f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5135076862361688, + "mae": 0.40329840779304504, + "r2": 0.5057573914527893, + "score": -0.5135076862361688 + }, + "val": { + "rmse": 0.5559787764083384, + "mae": 0.4406217336654663, + "r2": 0.41709965467453003, + "score": -0.5559787764083384 + }, + "test": { + "rmse": 0.5500968306007196, + "mae": 0.4433589279651642, + "r2": 0.4361191987991333, + "score": -0.5500968306007196 + } + }, + "time": "0:09:32.630353", + "diversity": { + "train": { + "mean_centered_corr": -0.02049259841442108, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05744750052690506 + }, + "val": { + "mean_centered_corr": -0.020656166598200798, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05917323753237724 + }, + "test": { + "mean_centered_corr": -0.020809270441532135, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05833838880062103 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..81009d573 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5135710584677824, + "mae": 0.40422096848487854, + "r2": 0.5056354403495789, + "score": -0.5135710584677824 + }, + "val": { + "rmse": 0.5553479091365543, + "mae": 0.44104957580566406, + "r2": 0.41842174530029297, + "score": -0.5553479091365543 + }, + "test": { + "rmse": 0.5511482044438852, + "mae": 0.4451128840446472, + "r2": 0.43396174907684326, + "score": -0.5511482044438852 + } + }, + "time": "0:09:32.630353", + "diversity": { + "train": { + "mean_centered_corr": -0.02049259841442108, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05744750052690506 + }, + "val": { + "mean_centered_corr": -0.020656166598200798, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05917323753237724 + }, + "test": { + "mean_centered_corr": -0.020809270441532135, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05833838880062103 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 14, + 29, + 15, + 11, + 19, + 31, + 21, + 2, + 15, + 19, + 22, + 29, + 15, + 26, + 13, + 8 + ], + "head_selection_time": "0:00:00.512283" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..19557f15f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5153977731507096, + "mae": 0.4045717418193817, + "r2": 0.5021123886108398, + "score": -0.5153977731507096 + }, + "val": { + "rmse": 0.5573103773832353, + "mae": 0.44244086742401123, + "r2": 0.41430413722991943, + "score": -0.5573103773832353 + }, + "test": { + "rmse": 0.5532468973961396, + "mae": 0.4463353753089905, + "r2": 0.4296427369117737, + "score": -0.5532468973961396 + } + }, + "time": "0:14:41.500927", + "diversity": { + "train": { + "mean_centered_corr": -0.02064315602183342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056059885770082474 + }, + "val": { + "mean_centered_corr": -0.020947575569152832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05900698900222778 + }, + "test": { + "mean_centered_corr": -0.020849451422691345, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05796251446008682 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.023481" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..9dd7c431e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..80b660117 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5133709914722272, + "mae": 0.4036697745323181, + "r2": 0.5060205459594727, + "score": -0.5133709914722272 + }, + "val": { + "rmse": 0.5556926485294449, + "mae": 0.4411713182926178, + "r2": 0.4176994562149048, + "score": -0.5556926485294449 + }, + "test": { + "rmse": 0.5502675684403805, + "mae": 0.4438808858394623, + "r2": 0.43576914072036743, + "score": -0.5502675684403805 + } + }, + "time": "0:14:41.500927", + "diversity": { + "train": { + "mean_centered_corr": -0.02064315602183342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056059885770082474 + }, + "val": { + "mean_centered_corr": -0.020947575569152832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05900698900222778 + }, + "test": { + "mean_centered_corr": -0.020849451422691345, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05796251446008682 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..855434aa0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,184 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5137274534378229, + "mae": 0.40429580211639404, + "r2": 0.5053342580795288, + "score": -0.5137274534378229 + }, + "val": { + "rmse": 0.5553051102575359, + "mae": 0.4410802721977234, + "r2": 0.4185113310813904, + "score": -0.5553051102575359 + }, + "test": { + "rmse": 0.5508335169889497, + "mae": 0.4448656141757965, + "r2": 0.43460792303085327, + "score": -0.5508335169889497 + } + }, + "time": "0:14:41.500927", + "diversity": { + "train": { + "mean_centered_corr": -0.02064315602183342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056059885770082474 + }, + "val": { + "mean_centered_corr": -0.020947575569152832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05900698900222778 + }, + "test": { + "mean_centered_corr": -0.020849451422691345, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05796251446008682 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 31, + 16, + 19, + 15, + 11, + 21 + ], + "head_selection_time": "0:00:00.177208" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..13d5060bf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5156084982803489, + "mae": 0.40666115283966064, + "r2": 0.5017051696777344, + "score": -0.5156084982803489 + }, + "val": { + "rmse": 0.5571872637155045, + "mae": 0.44240155816078186, + "r2": 0.4145628809928894, + "score": -0.5571872637155045 + }, + "test": { + "rmse": 0.5515561165588831, + "mae": 0.444357693195343, + "r2": 0.4331235885620117, + "score": -0.5515561165588831 + } + }, + "time": "0:14:43.936882", + "diversity": { + "train": { + "mean_centered_corr": -0.020564058795571327, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05423367768526077 + }, + "val": { + "mean_centered_corr": -0.020864037796854973, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05605003982782364 + }, + "test": { + "mean_centered_corr": -0.02069844678044319, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05536308139562607 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 10 + ], + "head_selection_time": "0:00:00.023846" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..7539c979e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..00fcfd604 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5131794414842274, + "mae": 0.40165284276008606, + "r2": 0.5063890218734741, + "score": -0.5131794414842274 + }, + "val": { + "rmse": 0.5564461619356259, + "mae": 0.4396740794181824, + "r2": 0.41611921787261963, + "score": -0.5564461619356259 + }, + "test": { + "rmse": 0.5491469465177617, + "mae": 0.44128578901290894, + "r2": 0.43806493282318115, + "score": -0.5491469465177617 + } + }, + "time": "0:14:43.936882", + "diversity": { + "train": { + "mean_centered_corr": -0.020564058795571327, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05423367768526077 + }, + "val": { + "mean_centered_corr": -0.020864037796854973, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05605003982782364 + }, + "test": { + "mean_centered_corr": -0.02069844678044319, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05536308139562607 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9c5138cde --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,190 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5133235315939391, + "mae": 0.4035474956035614, + "r2": 0.5061118602752686, + "score": -0.5133235315939391 + }, + "val": { + "rmse": 0.5552950473462183, + "mae": 0.44067850708961487, + "r2": 0.41853243112564087, + "score": -0.5552950473462183 + }, + "test": { + "rmse": 0.5512393912767233, + "mae": 0.44479161500930786, + "r2": 0.433774471282959, + "score": -0.5512393912767233 + } + }, + "time": "0:14:43.936882", + "diversity": { + "train": { + "mean_centered_corr": -0.020564058795571327, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05423367768526077 + }, + "val": { + "mean_centered_corr": -0.020864037796854973, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05605003982782364 + }, + "test": { + "mean_centered_corr": -0.02069844678044319, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05536308139562607 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 10, + 13, + 15, + 31, + 19, + 21, + 14, + 15, + 6, + 29, + 31, + 13 + ], + "head_selection_time": "0:00:00.354949" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..d3e620806 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.517674217660727, + "mae": 0.41062942147254944, + "r2": 0.4977044463157654, + "score": -0.517674217660727 + }, + "val": { + "rmse": 0.5569447272303437, + "mae": 0.4422590136528015, + "r2": 0.4150724411010742, + "score": -0.5569447272303437 + }, + "test": { + "rmse": 0.5529624818746817, + "mae": 0.446209579706192, + "r2": 0.4302290081977844, + "score": -0.5529624818746817 + } + }, + "time": "0:14:43.584777", + "diversity": { + "train": { + "mean_centered_corr": -0.020540937781333923, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05564684420824051 + }, + "val": { + "mean_centered_corr": -0.02083873376250267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05753690004348755 + }, + "test": { + "mean_centered_corr": -0.020769651979207993, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05645184963941574 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.023450" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..ae93dce05 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..f46125d6f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5139217276720119, + "mae": 0.4053196907043457, + "r2": 0.5049600601196289, + "score": -0.5139217276720119 + }, + "val": { + "rmse": 0.5560204514474498, + "mae": 0.4419814646244049, + "r2": 0.4170122742652893, + "score": -0.5560204514474498 + }, + "test": { + "rmse": 0.5515690033180755, + "mae": 0.4455237090587616, + "r2": 0.43309706449508667, + "score": -0.5515690033180755 + } + }, + "time": "0:14:43.584777", + "diversity": { + "train": { + "mean_centered_corr": -0.020540937781333923, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05564684420824051 + }, + "val": { + "mean_centered_corr": -0.02083873376250267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05753690004348755 + }, + "test": { + "mean_centered_corr": -0.020769651979207993, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05645184963941574 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b3e418d18 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,187 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5139979208874492, + "mae": 0.4061751067638397, + "r2": 0.5048133134841919, + "score": -0.5139979208874492 + }, + "val": { + "rmse": 0.5553112552523684, + "mae": 0.4416539669036865, + "r2": 0.4184984564781189, + "score": -0.5553112552523684 + }, + "test": { + "rmse": 0.552273210826067, + "mae": 0.4467461109161377, + "r2": 0.4316486120223999, + "score": -0.552273210826067 + } + }, + "time": "0:14:43.584777", + "diversity": { + "train": { + "mean_centered_corr": -0.020540937781333923, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05564684420824051 + }, + "val": { + "mean_centered_corr": -0.02083873376250267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05753690004348755 + }, + "test": { + "mean_centered_corr": -0.020769651979207993, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05645184963941574 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 29, + 10, + 26, + 19, + 11, + 15, + 21, + 13, + 31 + ], + "head_selection_time": "0:00:00.259576" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a2fb24fa6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 8830112, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5142542507513354, + "mae": 0.4036604166030884, + "r2": 0.5043192505836487, + "score": -0.5142542507513354 + }, + "val": { + "rmse": 0.5573390661266857, + "mae": 0.4425854980945587, + "r2": 0.41424381732940674, + "score": -0.5573390661266857 + }, + "test": { + "rmse": 0.5535791095517189, + "mae": 0.44649070501327515, + "r2": 0.42895758152008057, + "score": -0.5535791095517189 + } + }, + "time": "0:25:16.790683", + "diversity": { + "train": { + "mean_centered_corr": -0.02050994336605072, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0549476183950901 + }, + "val": { + "mean_centered_corr": -0.020625509321689606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056770239025354385 + }, + "test": { + "mean_centered_corr": -0.020527120679616928, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05573471263051033 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023311" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..a7ff216da --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..1078aa1c1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 8830112, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5118505584787533, + "mae": 0.39745861291885376, + "r2": 0.5089422464370728, + "score": -0.5118505584787533 + }, + "val": { + "rmse": 0.5572424061712593, + "mae": 0.43794944882392883, + "r2": 0.41444700956344604, + "score": -0.5572424061712593 + }, + "test": { + "rmse": 0.546917368746242, + "mae": 0.43608883023262024, + "r2": 0.4426186680793762, + "score": -0.546917368746242 + } + }, + "time": "0:25:16.790683", + "diversity": { + "train": { + "mean_centered_corr": -0.02050994336605072, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0549476183950901 + }, + "val": { + "mean_centered_corr": -0.020625509321689606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056770239025354385 + }, + "test": { + "mean_centered_corr": -0.020527120679616928, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05573471263051033 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8bc286149 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,185 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 8830112, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5124635881189222, + "mae": 0.4014242887496948, + "r2": 0.5077651739120483, + "score": -0.5124635881189222 + }, + "val": { + "rmse": 0.5558846409645507, + "mae": 0.440949946641922, + "r2": 0.41729700565338135, + "score": -0.5558846409645507 + }, + "test": { + "rmse": 0.5506559188769905, + "mae": 0.4435808062553406, + "r2": 0.43497246503829956, + "score": -0.5506559188769905 + } + }, + "time": "0:25:16.790683", + "diversity": { + "train": { + "mean_centered_corr": -0.02050994336605072, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0549476183950901 + }, + "val": { + "mean_centered_corr": -0.020625509321689606, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056770239025354385 + }, + "test": { + "mean_centered_corr": -0.020527120679616928, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05573471263051033 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 13, + 21, + 15, + 14, + 6, + 21, + 7 + ], + "head_selection_time": "0:00:00.201063" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..56c3d50b2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 8830112, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5108046192647389, + "mae": 0.3996152877807617, + "r2": 0.5109469890594482, + "score": -0.5108046192647389 + }, + "val": { + "rmse": 0.5576981787979653, + "mae": 0.4411272704601288, + "r2": 0.4134887456893921, + "score": -0.5576981787979653 + }, + "test": { + "rmse": 0.5514806270620619, + "mae": 0.4422125220298767, + "r2": 0.43327873945236206, + "score": -0.5514806270620619 + } + }, + "time": "0:26:26.010094", + "diversity": { + "train": { + "mean_centered_corr": -0.020078139379620552, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05760448798537254 + }, + "val": { + "mean_centered_corr": -0.0201308261603117, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05999069660902023 + }, + "test": { + "mean_centered_corr": -0.02021006867289543, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05763765797019005 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023898" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..cacd21376 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..13fcff92d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 8830112, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.5079285415698682, + "mae": 0.3954585790634155, + "r2": 0.5164387822151184, + "score": -0.5079285415698682 + }, + "val": { + "rmse": 0.5561609443354546, + "mae": 0.43867626786231995, + "r2": 0.41671764850616455, + "score": -0.5561609443354546 + }, + "test": { + "rmse": 0.5482846212731874, + "mae": 0.4379252791404724, + "r2": 0.43982839584350586, + "score": -0.5482846212731874 + } + }, + "time": "0:26:26.010094", + "diversity": { + "train": { + "mean_centered_corr": -0.020078139379620552, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05760448798537254 + }, + "val": { + "mean_centered_corr": -0.0201308261603117, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05999069660902023 + }, + "test": { + "mean_centered_corr": -0.02021006867289543, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05763765797019005 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..398031dcf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,186 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 8830112, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1638, + "metrics": { + "train": { + "rmse": 0.508383180826817, + "mae": 0.39700230956077576, + "r2": 0.5155726671218872, + "score": -0.508383180826817 + }, + "val": { + "rmse": 0.5556463362777673, + "mae": 0.43915992975234985, + "r2": 0.4177964925765991, + "score": -0.5556463362777673 + }, + "test": { + "rmse": 0.5491140035096563, + "mae": 0.4403056800365448, + "r2": 0.4381323456764221, + "score": -0.5491140035096563 + } + }, + "time": "0:26:26.010094", + "diversity": { + "train": { + "mean_centered_corr": -0.020078139379620552, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05760448798537254 + }, + "val": { + "mean_centered_corr": -0.0201308261603117, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05999069660902023 + }, + "test": { + "mean_centered_corr": -0.02021006867289543, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05763765797019005 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 13, + 4, + 21, + 25, + 15, + 0, + 1, + 17 + ], + "head_selection_time": "0:00:00.231882" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6ffea1cff --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 8830112, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5162090981285068, + "mae": 0.4077680706977844, + "r2": 0.5005435943603516, + "score": -0.5162090981285068 + }, + "val": { + "rmse": 0.5574517747950514, + "mae": 0.44307076930999756, + "r2": 0.41400688886642456, + "score": -0.5574517747950514 + }, + "test": { + "rmse": 0.5534907850292722, + "mae": 0.4466339945793152, + "r2": 0.42913979291915894, + "score": -0.5534907850292722 + } + }, + "time": "0:25:18.954003", + "diversity": { + "train": { + "mean_centered_corr": -0.020567411556839943, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056090690195560455 + }, + "val": { + "mean_centered_corr": -0.020773332566022873, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0600193589925766 + }, + "test": { + "mean_centered_corr": -0.020730366930365562, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058352041989564896 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 10 + ], + "head_selection_time": "0:00:00.023195" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..7baa58a72 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..6f7bf2e35 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 8830112, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5125091214647594, + "mae": 0.4022916555404663, + "r2": 0.5076777338981628, + "score": -0.5125091214647594 + }, + "val": { + "rmse": 0.5558005167077797, + "mae": 0.44092121720314026, + "r2": 0.41747337579727173, + "score": -0.5558005167077797 + }, + "test": { + "rmse": 0.5500678724978622, + "mae": 0.4430854618549347, + "r2": 0.43617862462997437, + "score": -0.5500678724978622 + } + }, + "time": "0:25:18.954003", + "diversity": { + "train": { + "mean_centered_corr": -0.020567411556839943, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056090690195560455 + }, + "val": { + "mean_centered_corr": -0.020773332566022873, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0600193589925766 + }, + "test": { + "mean_centered_corr": -0.020730366930365562, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058352041989564896 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c28c6f984 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,187 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 8830112, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5134856027741057, + "mae": 0.40518614649772644, + "r2": 0.5057998895645142, + "score": -0.5134856027741057 + }, + "val": { + "rmse": 0.5553793554897237, + "mae": 0.44185882806777954, + "r2": 0.4183558225631714, + "score": -0.5553793554897237 + }, + "test": { + "rmse": 0.5521969288494308, + "mae": 0.4467563331127167, + "r2": 0.4318056106567383, + "score": -0.5521969288494308 + } + }, + "time": "0:25:18.954003", + "diversity": { + "train": { + "mean_centered_corr": -0.020567411556839943, + "mean_pairwise_disagreement": NaN, + "member_std": 0.056090690195560455 + }, + "val": { + "mean_centered_corr": -0.020773332566022873, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0600193589925766 + }, + "test": { + "mean_centered_corr": -0.020730366930365562, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058352041989564896 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 10, + 22, + 31, + 21, + 29, + 7, + 15, + 16, + 13 + ], + "head_selection_time": "0:00:00.257032" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..02eadaaea --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-best-head-evaluation/0/report.json @@ -0,0 +1,445 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.788873839009288, + "recall": 0.9686263795008256, + "f1-score": 0.8695577986445486, + "support": 84179.0 + }, + "1": { + "precision": 0.5584350443069721, + "recall": 0.13273984579922105, + "f1-score": 0.21449442892463796, + "support": 25162.0 + }, + "accuracy": 0.7762687372531805, + "macro avg": { + "precision": 0.67365444165813, + "recall": 0.5506831126500233, + "f1-score": 0.5420261137845933, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7358443171254596, + "recall": 0.7762687372531805, + "f1-score": 0.7188119255604138, + "support": 109341.0 + }, + "cross-entropy": 0.4989015904618268, + "roc-auc": 0.6823993201798577, + "score": 0.6823993201798577 + }, + "val": { + "0": { + "precision": 0.6940704108113401, + "recall": 0.9134553421832711, + "f1-score": 0.7887927436632258, + "support": 15518.0 + }, + "1": { + "precision": 0.6500781657113079, + "recall": 0.28537115406611, + "f1-score": 0.39662983864557666, + "support": 8743.0 + }, + "accuracy": 0.6871110011953341, + "macro avg": { + "precision": 0.672074288261324, + "recall": 0.5994132481246905, + "f1-score": 0.5927112911544012, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.678216810427614, + "recall": 0.6871110011953341, + "f1-score": 0.6474679722783157, + "support": 24261.0 + }, + "cross-entropy": 0.6244923069797662, + "roc-auc": 0.6479868261151002, + "score": 0.6479868261151002 + }, + "test": { + "0": { + "precision": 0.6674781317049018, + "recall": 0.9559744711027065, + "f1-score": 0.7860926186889547, + "support": 16922.0 + }, + "1": { + "precision": 0.6642631816133393, + "recall": 0.15462079093674605, + "f1-score": 0.25085091899251194, + "support": 9533.0 + }, + "accuracy": 0.6672084672084672, + "macro avg": { + "precision": 0.6658706566591206, + "recall": 0.5552976310197263, + "f1-score": 0.5184717688407333, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6663196316397774, + "recall": 0.6672084672084672, + "f1-score": 0.593219470958613, + "support": 26455.0 + }, + "cross-entropy": 0.65875352245754, + "roc-auc": 0.6047175337399694, + "score": 0.6047175337399694 + } + }, + "time": "0:03:40.582211", + "diversity": { + "train": { + "mean_centered_corr": -0.010312517173588276, + "mean_pairwise_disagreement": 0.02666120948639223, + "member_std": 0.029578423127532005 + }, + "val": { + "mean_centered_corr": -0.012083624489605427, + "mean_pairwise_disagreement": 0.09328301030061521, + "member_std": 0.06631172448396683 + }, + "test": { + "mean_centered_corr": -0.010351695120334625, + "mean_pairwise_disagreement": 0.058351141012431336, + "member_std": 0.06402193009853363 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 22 + ], + "head_selection_time": "0:00:04.820494" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..b8da71126 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0.toml @@ -0,0 +1,74 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..81bd76de1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/0/report.json @@ -0,0 +1,441 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7843377382851067, + "recall": 0.9814680621057509, + "f1-score": 0.8718993219534074, + "support": 84179.0 + }, + "1": { + "precision": 0.6104868913857678, + "recall": 0.09717033622128607, + "f1-score": 0.16765522679740802, + "support": 25162.0 + }, + "accuracy": 0.7779698374809083, + "macro avg": { + "precision": 0.6974123148354372, + "recall": 0.5393191991635184, + "f1-score": 0.5197772743754077, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7443304673649471, + "recall": 0.7779698374809083, + "f1-score": 0.7098357783392529, + "support": 109341.0 + }, + "cross-entropy": 0.49273880116728924, + "roc-auc": 0.6982248815909875, + "score": 0.6982248815909875 + }, + "val": { + "0": { + "precision": 0.6767916685698625, + "recall": 0.9548266529191907, + "f1-score": 0.7921199647162599, + "support": 15518.0 + }, + "1": { + "precision": 0.7039695945945946, + "recall": 0.19066681916962142, + "f1-score": 0.3000630006300063, + "support": 8743.0 + }, + "accuracy": 0.6794443757470838, + "macro avg": { + "precision": 0.6903806315822285, + "recall": 0.5727467360444061, + "f1-score": 0.5460914826731331, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6865858488276521, + "recall": 0.6794443757470838, + "f1-score": 0.6147961100933624, + "support": 24261.0 + }, + "cross-entropy": 0.6764607634104041, + "roc-auc": 0.6244437709503304, + "score": 0.6244437709503304 + }, + "test": { + "0": { + "precision": 0.657613234183004, + "recall": 0.9772485521805933, + "f1-score": 0.7861846014880316, + "support": 16922.0 + }, + "1": { + "precision": 0.7056574923547401, + "recall": 0.09682156718766391, + "f1-score": 0.17027949451157642, + "support": 9533.0 + }, + "accuracy": 0.65998865998866, + "macro avg": { + "precision": 0.6816353632688721, + "recall": 0.5370350596841286, + "f1-score": 0.478232047999804, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6749258750127587, + "recall": 0.65998865998866, + "f1-score": 0.5642445756023182, + "support": 26455.0 + }, + "cross-entropy": 0.7397410477591574, + "roc-auc": 0.592244305956134, + "score": 0.592244305956134 + } + }, + "time": "0:03:40.582211", + "diversity": { + "train": { + "mean_centered_corr": -0.010312517173588276, + "mean_pairwise_disagreement": 0.02666120948639223, + "member_std": 0.029578423127532005 + }, + "val": { + "mean_centered_corr": -0.012083624489605427, + "mean_pairwise_disagreement": 0.09328301030061521, + "member_std": 0.06631172448396683 + }, + "test": { + "mean_centered_corr": -0.010351695120334625, + "mean_pairwise_disagreement": 0.058351141012431336, + "member_std": 0.06402193009853363 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9e8350c7b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,447 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7909716267830696, + "recall": 0.9643616578956747, + "f1-score": 0.8691029971468489, + "support": 84179.0 + }, + "1": { + "precision": 0.5528394693695037, + "recall": 0.1474048167872188, + "f1-score": 0.2327507765680399, + "support": 25162.0 + }, + "accuracy": 0.7763601942546712, + "macro avg": { + "precision": 0.6719055480762867, + "recall": 0.5558832373414467, + "f1-score": 0.5509268868574444, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.736171676674326, + "recall": 0.7763601942546712, + "f1-score": 0.7226630105525798, + "support": 109341.0 + }, + "cross-entropy": 0.5000760433868655, + "roc-auc": 0.6809180604528166, + "score": 0.6809180604528166 + }, + "val": { + "0": { + "precision": 0.7073092712054166, + "recall": 0.8549426472483568, + "f1-score": 0.7741502552881109, + "support": 15518.0 + }, + "1": { + "precision": 0.5910247093023255, + "recall": 0.37206908383849935, + "f1-score": 0.45665754193865377, + "support": 8743.0 + }, + "accuracy": 0.6809282387370678, + "macro avg": { + "precision": 0.649166990253871, + "recall": 0.613505865543428, + "f1-score": 0.6154038986133823, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6654034996082555, + "recall": 0.6809282387370678, + "f1-score": 0.6597345760986997, + "support": 24261.0 + }, + "cross-entropy": 0.6204112258943162, + "roc-auc": 0.6505096994576862, + "score": 0.6505096994576862 + }, + "test": { + "0": { + "precision": 0.6799947561615102, + "recall": 0.9195721545916559, + "f1-score": 0.7818419333768779, + "support": 16922.0 + }, + "1": { + "precision": 0.6188742649117894, + "recall": 0.2318262876324347, + "f1-score": 0.3373015873015873, + "support": 9533.0 + }, + "accuracy": 0.6717444717444717, + "macro avg": { + "precision": 0.6494345105366498, + "recall": 0.5756992211120453, + "f1-score": 0.5595717603392326, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6579701240283184, + "recall": 0.6717444717444717, + "f1-score": 0.6216528152844286, + "support": 26455.0 + }, + "cross-entropy": 0.6441552092220911, + "roc-auc": 0.6063775930816054, + "score": 0.6063775930816054 + } + }, + "time": "0:03:40.582211", + "diversity": { + "train": { + "mean_centered_corr": -0.010312517173588276, + "mean_pairwise_disagreement": 0.02666120948639223, + "member_std": 0.029578423127532005 + }, + "val": { + "mean_centered_corr": -0.012083624489605427, + "mean_pairwise_disagreement": 0.09328301030061521, + "member_std": 0.06631172448396683 + }, + "test": { + "mean_centered_corr": -0.010351695120334625, + "mean_pairwise_disagreement": 0.058351141012431336, + "member_std": 0.06402193009853363 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 22, + 24, + 24 + ], + "head_selection_time": "0:00:20.539821" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..fb5a78bba --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,274 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.791864764923402, + "recall": 0.9615937466589054, + "f1-score": 0.8685146538913417, + "support": 84179.0 + }, + "1": { + "precision": 0.5458631830313246, + "recall": 0.1544392337652015, + "f1-score": 0.24076081905765, + "support": 25162.0 + }, + "accuracy": 0.7758480350463229, + "macro avg": { + "precision": 0.6688639739773633, + "recall": 0.5580164902120535, + "f1-score": 0.5546377364744959, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7352538705327485, + "recall": 0.7758480350463229, + "f1-score": 0.7240533631396078, + "support": 109341.0 + }, + "cross-entropy": 0.5019628647078745, + "roc-auc": 0.677748911462424, + "score": 0.677748911462424 + }, + "val": { + "0": { + "precision": 0.709374145941514, + "recall": 0.8363191132877948, + "f1-score": 0.7676337503327123, + "support": 15518.0 + }, + "1": { + "precision": 0.5742541066040898, + "recall": 0.39185634221663046, + "f1-score": 0.465837242504589, + "support": 8743.0 + }, + "accuracy": 0.6761469024360084, + "macro avg": { + "precision": 0.6418141262728019, + "recall": 0.6140877277522127, + "f1-score": 0.6167354964186507, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.660680584096285, + "recall": 0.6761469024360084, + "f1-score": 0.6588745537645048, + "support": 24261.0 + }, + "cross-entropy": 0.6230257230038371, + "roc-auc": 0.6498655113216565, + "score": 0.6498655113216565 + }, + "test": { + "0": { + "precision": 0.6823990382474732, + "recall": 0.9056849072213686, + "f1-score": 0.7783448871283106, + "support": 16922.0 + }, + "1": { + "precision": 0.6006006006006006, + "recall": 0.25175705444246305, + "f1-score": 0.3547934067558578, + "support": 9533.0 + }, + "accuracy": 0.6700434700434701, + "macro avg": { + "precision": 0.6414998194240369, + "recall": 0.5787209808319158, + "f1-score": 0.5665691469420842, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6529231544414767, + "recall": 0.6700434700434701, + "f1-score": 0.6257190597841189, + "support": 26455.0 + }, + "cross-entropy": 0.6443544898598349, + "roc-auc": 0.6047694345185003, + "score": 0.6047694345185003 + } + }, + "time": "0:03:18.117749", + "diversity": { + "train": { + "mean_centered_corr": -0.011164821684360504, + "mean_pairwise_disagreement": 0.026747282325698445, + "member_std": 0.029834141954779625 + }, + "val": { + "mean_centered_corr": -0.007544255815446377, + "mean_pairwise_disagreement": 0.09209897804919884, + "member_std": 0.06599097698926926 + }, + "test": { + "mean_centered_corr": -0.005217556841671467, + "mean_pairwise_disagreement": 0.057501402259466775, + "member_std": 0.06378328055143356 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 24 + ], + "head_selection_time": "0:00:04.771288" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..972126362 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0.toml @@ -0,0 +1,63 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..51c1a0f19 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/0/report.json @@ -0,0 +1,270 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7843520689556971, + "recall": 0.9815512182373276, + "f1-score": 0.8719409883812961, + "support": 84179.0 + }, + "1": { + "precision": 0.6115557778889444, + "recall": 0.09717033622128607, + "f1-score": 0.16769547325102882, + "support": 25162.0 + }, + "accuracy": 0.7780338573819519, + "macro avg": { + "precision": 0.6979539234223208, + "recall": 0.5393607772293069, + "f1-score": 0.5198182308161624, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7445874767549523, + "recall": 0.7780338573819519, + "f1-score": 0.7098771179968311, + "support": 109341.0 + }, + "cross-entropy": 0.4926987595087899, + "roc-auc": 0.69829445109446, + "score": 0.69829445109446 + }, + "val": { + "0": { + "precision": 0.6763216713041098, + "recall": 0.9554710658590024, + "f1-score": 0.7920194439251088, + "support": 15518.0 + }, + "1": { + "precision": 0.704448246364414, + "recall": 0.18837927484845018, + "f1-score": 0.29726558974821765, + "support": 8743.0 + }, + "accuracy": 0.6790321915831994, + "macro avg": { + "precision": 0.6903849588342619, + "recall": 0.5719251703537263, + "f1-score": 0.5446425168366632, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6864577186950763, + "recall": 0.6790321915831994, + "f1-score": 0.6137237039692719, + "support": 24261.0 + }, + "cross-entropy": 0.6773915501327039, + "roc-auc": 0.6244116424360375, + "score": 0.6244116424360375 + }, + "test": { + "0": { + "precision": 0.6572393247269116, + "recall": 0.9777804042075405, + "f1-score": 0.7860892700192412, + "support": 16922.0 + }, + "1": { + "precision": 0.70625, + "recall": 0.09482849050666108, + "f1-score": 0.1672061407564968, + "support": 9533.0 + }, + "accuracy": 0.6596106596106596, + "macro avg": { + "precision": 0.6817446623634558, + "recall": 0.5363044473571008, + "f1-score": 0.47664770538786905, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6749002117947003, + "recall": 0.6596106596106596, + "f1-score": 0.5630761204723979, + "support": 26455.0 + }, + "cross-entropy": 0.7412929728578691, + "roc-auc": 0.5923170352346188, + "score": 0.5923170352346188 + } + }, + "time": "0:03:18.117749", + "diversity": { + "train": { + "mean_centered_corr": -0.011164821684360504, + "mean_pairwise_disagreement": 0.026747282325698445, + "member_std": 0.029834141954779625 + }, + "val": { + "mean_centered_corr": -0.007544255815446377, + "mean_pairwise_disagreement": 0.09209897804919884, + "member_std": 0.06599097698926926 + }, + "test": { + "mean_centered_corr": -0.005217556841671467, + "mean_pairwise_disagreement": 0.057501402259466775, + "member_std": 0.06378328055143356 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..e7c657dd9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7903111430597515, + "recall": 0.966476199527198, + "f1-score": 0.869561035046654, + "support": 84179.0 + }, + "1": { + "precision": 0.5589246639574867, + "recall": 0.14211906843653127, + "f1-score": 0.22661596958174904, + "support": 25162.0 + }, + "accuracy": 0.7767717507613795, + "macro avg": { + "precision": 0.674617903508619, + "recall": 0.5542976339818646, + "f1-score": 0.5480885023142016, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.737063536149524, + "recall": 0.7767717507613795, + "f1-score": 0.7216038759093868, + "support": 109341.0 + }, + "cross-entropy": 0.49896313282151494, + "roc-auc": 0.6832990228404343, + "score": 0.6832990228404343 + }, + "val": { + "0": { + "precision": 0.7028659160696008, + "recall": 0.8850367315375692, + "f1-score": 0.7835016258771179, + "support": 15518.0 + }, + "1": { + "precision": 0.6221139589070113, + "recall": 0.33592588356399405, + "f1-score": 0.4362745098039216, + "support": 8743.0 + }, + "accuracy": 0.6871522196117226, + "macro avg": { + "precision": 0.662489937488306, + "recall": 0.6104813075507817, + "f1-score": 0.6098880678405197, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6737651221422063, + "recall": 0.6871522196117226, + "f1-score": 0.6583704822380281, + "support": 24261.0 + }, + "cross-entropy": 0.6202643229081582, + "roc-auc": 0.6505072081895442, + "score": 0.6505072081895442 + }, + "test": { + "0": { + "precision": 0.6762482381582882, + "recall": 0.9356459047393925, + "f1-score": 0.7850749969009545, + "support": 16922.0 + }, + "1": { + "precision": 0.6420118343195266, + "recall": 0.2048673030525543, + "f1-score": 0.31061630218687875, + "support": 9533.0 + }, + "accuracy": 0.6723114723114723, + "macro avg": { + "precision": 0.6591300362389074, + "recall": 0.5702566038959733, + "f1-score": 0.5478456495439166, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6639112267126289, + "recall": 0.6723114723114723, + "f1-score": 0.6141048688832155, + "support": 26455.0 + }, + "cross-entropy": 0.6493383320901536, + "roc-auc": 0.6062618182365493, + "score": 0.6062618182365493 + } + }, + "time": "0:03:18.117749", + "diversity": { + "train": { + "mean_centered_corr": -0.011164821684360504, + "mean_pairwise_disagreement": 0.026747282325698445, + "member_std": 0.029834141954779625 + }, + "val": { + "mean_centered_corr": -0.007544255815446377, + "mean_pairwise_disagreement": 0.09209897804919884, + "member_std": 0.06599097698926926 + }, + "test": { + "mean_centered_corr": -0.005217556841671467, + "mean_pairwise_disagreement": 0.057501402259466775, + "member_std": 0.06378328055143356 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 24, + 22 + ], + "head_selection_time": "0:00:15.900575" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1712e7390 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-best-head-evaluation/0/report.json @@ -0,0 +1,435 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7914072107595307, + "recall": 0.9632568692904406, + "f1-score": 0.8689166671131662, + "support": 84179.0 + }, + "1": { + "precision": 0.5506319918640128, + "recall": 0.15062395676019394, + "f1-score": 0.2365423623030114, + "support": 25162.0 + }, + "accuracy": 0.7762504458528823, + "macro avg": { + "precision": 0.6710196013117717, + "recall": 0.5569404130253173, + "f1-score": 0.5527295147080888, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7359990284871075, + "recall": 0.7762504458528823, + "f1-score": 0.7233920948334804, + "support": 109341.0 + }, + "cross-entropy": 0.5022261934082275, + "roc-auc": 0.6755481612167327, + "score": 0.6755481612167327 + }, + "val": { + "0": { + "precision": 0.7100723427935448, + "recall": 0.8222709111998969, + "f1-score": 0.7620640229335881, + "support": 15518.0 + }, + "1": { + "precision": 0.5615959306946431, + "recall": 0.40409470433489647, + "f1-score": 0.470001330317946, + "support": 8743.0 + }, + "accuracy": 0.6715716582168914, + "macro avg": { + "precision": 0.635834136744094, + "recall": 0.6131828077673966, + "f1-score": 0.6160326766257671, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6565655099762373, + "recall": 0.6715716582168914, + "f1-score": 0.656812626802408, + "support": 24261.0 + }, + "cross-entropy": 0.6242117770764835, + "roc-auc": 0.6476721782117019, + "score": 0.6476721782117019 + }, + "test": { + "0": { + "precision": 0.6831312812654919, + "recall": 0.8957570027183548, + "f1-score": 0.775127202065915, + "support": 16922.0 + }, + "1": { + "precision": 0.5864978902953587, + "recall": 0.26245672925626773, + "f1-score": 0.36263497354880786, + "support": 9533.0 + }, + "accuracy": 0.6675486675486676, + "macro avg": { + "precision": 0.6348145857804253, + "recall": 0.5791068659873113, + "f1-score": 0.5688810878073615, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6483096552545949, + "recall": 0.6675486675486676, + "f1-score": 0.626486551358919, + "support": 26455.0 + }, + "cross-entropy": 0.6431486794584158, + "roc-auc": 0.6040083543113315, + "score": 0.6040083543113315 + } + }, + "time": "0:02:18.381792", + "diversity": { + "train": { + "mean_centered_corr": -0.010318981483578682, + "mean_pairwise_disagreement": 0.026643028719563627, + "member_std": 0.029531365260481834 + }, + "val": { + "mean_centered_corr": -0.01181983295828104, + "mean_pairwise_disagreement": 0.09317281751809289, + "member_std": 0.06609699130058289 + }, + "test": { + "mean_centered_corr": -0.010081159882247448, + "mean_pairwise_disagreement": 0.05819193883710013, + "member_std": 0.06373059004545212 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 24 + ], + "head_selection_time": "0:00:04.830833" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0.toml new file mode 100644 index 000000000..772f87508 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0.toml @@ -0,0 +1,64 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0/report.json new file mode 100644 index 000000000..5af8eb5f7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-evaluation/0/report.json @@ -0,0 +1,431 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7843261973702947, + "recall": 0.9814443032110146, + "f1-score": 0.8718828160452526, + "support": 84179.0 + }, + "1": { + "precision": 0.6100848726909636, + "recall": 0.09713059375248391, + "f1-score": 0.1675809105869446, + "support": 25162.0 + }, + "accuracy": 0.7779424003804611, + "macro avg": { + "precision": 0.6972055350306292, + "recall": 0.5392874484817493, + "f1-score": 0.5197318633160987, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7442290680996522, + "recall": 0.7779424003804611, + "f1-score": 0.7098059688868954, + "support": 109341.0 + }, + "cross-entropy": 0.49269774798365623, + "roc-auc": 0.6983158737104703, + "score": 0.6983158737104703 + }, + "val": { + "0": { + "precision": 0.6767916685698625, + "recall": 0.9548266529191907, + "f1-score": 0.7921199647162599, + "support": 15518.0 + }, + "1": { + "precision": 0.7039695945945946, + "recall": 0.19066681916962142, + "f1-score": 0.3000630006300063, + "support": 8743.0 + }, + "accuracy": 0.6794443757470838, + "macro avg": { + "precision": 0.6903806315822285, + "recall": 0.5727467360444061, + "f1-score": 0.5460914826731331, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6865858488276521, + "recall": 0.6794443757470838, + "f1-score": 0.6147961100933624, + "support": 24261.0 + }, + "cross-entropy": 0.6766578943018672, + "roc-auc": 0.6245232151327823, + "score": 0.6245232151327823 + }, + "test": { + "0": { + "precision": 0.6575609368165732, + "recall": 0.9772485521805933, + "f1-score": 0.7861472273062204, + "support": 16922.0 + }, + "1": { + "precision": 0.7052067381316999, + "recall": 0.09661176964229519, + "f1-score": 0.16994187655687795, + "support": 9533.0 + }, + "accuracy": 0.6599130599130599, + "macro avg": { + "precision": 0.6813838374741366, + "recall": 0.5369301609114442, + "f1-score": 0.4780445519315492, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6747299946104535, + "recall": 0.6599130599130599, + "f1-score": 0.5640990092493887, + "support": 26455.0 + }, + "cross-entropy": 0.7401458486738616, + "roc-auc": 0.5923207050179439, + "score": 0.5923207050179439 + } + }, + "time": "0:02:18.381792", + "diversity": { + "train": { + "mean_centered_corr": -0.010318981483578682, + "mean_pairwise_disagreement": 0.026643028719563627, + "member_std": 0.029531365260481834 + }, + "val": { + "mean_centered_corr": -0.01181983295828104, + "mean_pairwise_disagreement": 0.09317281751809289, + "member_std": 0.06609699130058289 + }, + "test": { + "mean_centered_corr": -0.010081159882247448, + "mean_pairwise_disagreement": 0.05819193883710013, + "member_std": 0.06373059004545212 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1cc02fd88 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_esam-greedy-heads-evaluation/0/report.json @@ -0,0 +1,436 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.790500918983575, + "recall": 0.9656565176587986, + "f1-score": 0.8693438853537244, + "support": 84179.0 + }, + "1": { + "precision": 0.5559139784946237, + "recall": 0.14382799459502424, + "f1-score": 0.22852993180095985, + "support": 25162.0 + }, + "accuracy": 0.7765339625575036, + "macro avg": { + "precision": 0.6732074487390993, + "recall": 0.5547422561269114, + "f1-score": 0.5489369085773421, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.736516808754265, + "recall": 0.7765339625575036, + "f1-score": 0.7218771464424774, + "support": 109341.0 + }, + "cross-entropy": 0.4993286461130596, + "roc-auc": 0.6825283693048605, + "score": 0.6825283693048605 + }, + "val": { + "0": { + "precision": 0.7052691926825455, + "recall": 0.8720195901533703, + "f1-score": 0.7798299956778563, + "support": 15518.0 + }, + "1": { + "precision": 0.6085928261726449, + "recall": 0.35319684318883676, + "f1-score": 0.4469855974524137, + "support": 8743.0 + }, + "accuracy": 0.685050080375912, + "macro avg": { + "precision": 0.6569310094275952, + "recall": 0.6126082166711035, + "f1-score": 0.613407796565135, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6704296777245445, + "recall": 0.685050080375912, + "f1-score": 0.6598819896729494, + "support": 24261.0 + }, + "cross-entropy": 0.6199389736027974, + "roc-auc": 0.6504598446123828, + "score": 0.6504598446123828 + }, + "test": { + "0": { + "precision": 0.6781034482758621, + "recall": 0.929677343103652, + "f1-score": 0.7842081650964559, + "support": 16922.0 + }, + "1": { + "precision": 0.6344086021505376, + "recall": 0.21661596559320256, + "f1-score": 0.32295902408507976, + "support": 9533.0 + }, + "accuracy": 0.6727272727272727, + "macro avg": { + "precision": 0.6562560252131999, + "recall": 0.5731466543484273, + "f1-score": 0.5535835945907678, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6623581083358614, + "recall": 0.6727272727272727, + "f1-score": 0.6179980701706782, + "support": 26455.0 + }, + "cross-entropy": 0.6464247215340417, + "roc-auc": 0.6064818006704371, + "score": 0.6064818006704371 + } + }, + "time": "0:02:18.381792", + "diversity": { + "train": { + "mean_centered_corr": -0.010318981483578682, + "mean_pairwise_disagreement": 0.026643028719563627, + "member_std": 0.029531365260481834 + }, + "val": { + "mean_centered_corr": -0.01181983295828104, + "mean_pairwise_disagreement": 0.09317281751809289, + "member_std": 0.06609699130058289 + }, + "test": { + "mean_centered_corr": -0.010081159882247448, + "mean_pairwise_disagreement": 0.05819193883710013, + "member_std": 0.06373059004545212 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 24, + 22 + ], + "head_selection_time": "0:00:15.884778" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ba2bc2674 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,409 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7819656172915348, + "recall": 0.9828935957899239, + "f1-score": 0.8709918521169758, + "support": 84179.0 + }, + "1": { + "precision": 0.5922989807474519, + "recall": 0.08314124473412289, + "f1-score": 0.14581445598382936, + "support": 25162.0 + }, + "accuracy": 0.7758388893461738, + "macro avg": { + "precision": 0.6871322990194934, + "recall": 0.5330174202620234, + "f1-score": 0.5084031540504026, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7383187519004901, + "recall": 0.7758388893461738, + "f1-score": 0.7041110513057317, + "support": 109341.0 + }, + "cross-entropy": 0.500751581030381, + "roc-auc": 0.6780775151909603, + "score": 0.6780775151909603 + }, + "val": { + "0": { + "precision": 0.6903867726674145, + "recall": 0.9121665163036474, + "f1-score": 0.7859304294716971, + "support": 15518.0 + }, + "1": { + "precision": 0.6373070782331027, + "recall": 0.2739334324602539, + "f1-score": 0.38316934645228384, + "support": 8743.0 + }, + "accuracy": 0.682164791228721, + "macro avg": { + "precision": 0.6638469254502586, + "recall": 0.5930499743819506, + "f1-score": 0.5845498879619905, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6712583044081017, + "recall": 0.682164791228721, + "f1-score": 0.6407863649715229, + "support": 24261.0 + }, + "cross-entropy": 0.6287444440087023, + "roc-auc": 0.637709740638791, + "score": 0.637709740638791 + }, + "test": { + "0": { + "precision": 0.6690629440775725, + "recall": 0.945987471930032, + "f1-score": 0.7837837837837838, + "support": 16922.0 + }, + "1": { + "precision": 0.638592328983788, + "recall": 0.16941151788524075, + "f1-score": 0.2677831205438567, + "support": 9533.0 + }, + "accuracy": 0.6661500661500661, + "macro avg": { + "precision": 0.6538276365306803, + "recall": 0.5576994949076364, + "f1-score": 0.5257834521638203, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6580829261721086, + "recall": 0.6661500661500661, + "f1-score": 0.5978441004473171, + "support": 26455.0 + }, + "cross-entropy": 0.6564068211877354, + "roc-auc": 0.6004564813723224, + "score": 0.6004564813723224 + } + }, + "time": "0:03:16.099007", + "diversity": { + "train": { + "mean_centered_corr": -0.01766625978052616, + "mean_pairwise_disagreement": 0.024688098434875683, + "member_std": 0.027821816504001617 + }, + "val": { + "mean_centered_corr": -0.004993404261767864, + "mean_pairwise_disagreement": 0.06782124769476035, + "member_std": 0.05843981355428696 + }, + "test": { + "mean_centered_corr": 0.0002984397578984499, + "mean_pairwise_disagreement": 0.04484334323044, + "member_std": 0.059851765632629395 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14 + ], + "head_selection_time": "0:00:04.874514" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0.toml new file mode 100644 index 000000000..8d636ee4c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0.toml @@ -0,0 +1,61 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0/report.json new file mode 100644 index 000000000..5477da392 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/0/report.json @@ -0,0 +1,405 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7813800636546827, + "recall": 0.9857684220530061, + "f1-score": 0.8717544661382415, + "support": 84179.0 + }, + "1": { + "precision": 0.6188355074769328, + "recall": 0.07729910182020507, + "f1-score": 0.13743154919625508, + "support": 25162.0 + }, + "accuracy": 0.7767077308603361, + "macro avg": { + "precision": 0.7001077855658078, + "recall": 0.5315337619366056, + "f1-score": 0.5045930076672482, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439746427920186, + "recall": 0.7767077308603361, + "f1-score": 0.7027690605164321, + "support": 109341.0 + }, + "cross-entropy": 0.4957520792555438, + "roc-auc": 0.6913928653833158, + "score": 0.6913928653833158 + }, + "val": { + "0": { + "precision": 0.6750442317288935, + "recall": 0.9588864544400052, + "f1-score": 0.7923111738239131, + "support": 15518.0 + }, + "1": { + "precision": 0.7123534715960325, + "recall": 0.18071600137252658, + "f1-score": 0.2882948636073351, + "support": 8743.0 + }, + "accuracy": 0.6784551337537612, + "macro avg": { + "precision": 0.6936988516624629, + "recall": 0.5698012279062659, + "f1-score": 0.5403030187156241, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6884894600442307, + "recall": 0.6784551337537612, + "f1-score": 0.6106774983685097, + "support": 24261.0 + }, + "cross-entropy": 0.6741937219356421, + "roc-auc": 0.6226455728683622, + "score": 0.6226455728683622 + }, + "test": { + "0": { + "precision": 0.6571859256614702, + "recall": 0.9790213922704172, + "f1-score": 0.7864517813486507, + "support": 16922.0 + }, + "1": { + "precision": 0.7150882825040128, + "recall": 0.0934648064617644, + "f1-score": 0.16532145839131646, + "support": 9533.0 + }, + "accuracy": 0.6599130599130599, + "macro avg": { + "precision": 0.6861371040827415, + "recall": 0.5362430993660908, + "f1-score": 0.4758866198699836, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6780509102685373, + "recall": 0.6599130599130599, + "f1-score": 0.5626288605868942, + "support": 26455.0 + }, + "cross-entropy": 0.7346957668444309, + "roc-auc": 0.5910976319446108, + "score": 0.5910976319446108 + } + }, + "time": "0:03:16.099007", + "diversity": { + "train": { + "mean_centered_corr": -0.01766625978052616, + "mean_pairwise_disagreement": 0.024688098434875683, + "member_std": 0.027821816504001617 + }, + "val": { + "mean_centered_corr": -0.004993404261767864, + "mean_pairwise_disagreement": 0.06782124769476035, + "member_std": 0.05843981355428696 + }, + "test": { + "mean_centered_corr": 0.0002984397578984499, + "mean_pairwise_disagreement": 0.04484334323044, + "member_std": 0.059851765632629395 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3a857ba75 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,411 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7814865438092096, + "recall": 0.9848537046056618, + "f1-score": 0.8714628095698608, + "support": 84179.0 + }, + "1": { + "precision": 0.6084152334152334, + "recall": 0.0787298306970829, + "f1-score": 0.13941867830248433, + "support": 25162.0 + }, + "accuracy": 0.7763327571542239, + "macro avg": { + "precision": 0.6949508886122215, + "recall": 0.5317917676513724, + "f1-score": 0.5054407439361726, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7416586630313383, + "recall": 0.7763327571542239, + "f1-score": 0.7030018074668096, + "support": 109341.0 + }, + "cross-entropy": 0.49890429497744304, + "roc-auc": 0.6841500123073285, + "score": 0.6841500123073285 + }, + "val": { + "0": { + "precision": 0.6818666295935863, + "recall": 0.9481891996391287, + "f1-score": 0.7932716931288244, + "support": 15518.0 + }, + "1": { + "precision": 0.7002237136465325, + "recall": 0.21480041175797782, + "f1-score": 0.3287527352297593, + "support": 8743.0 + }, + "accuracy": 0.6838959647170356, + "macro avg": { + "precision": 0.6910451716200594, + "recall": 0.5814948056985533, + "f1-score": 0.5610122141792918, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6884820199680519, + "recall": 0.6838959647170356, + "f1-score": 0.6258717817932848, + "support": 24261.0 + }, + "cross-entropy": 0.6354781186535776, + "roc-auc": 0.6386704340734016, + "score": 0.6386704340734016 + }, + "test": { + "0": { + "precision": 0.663352330140253, + "recall": 0.9698617184729937, + "f1-score": 0.7878453303890742, + "support": 16922.0 + }, + "1": { + "precision": 0.7024504084014003, + "recall": 0.12629812231196896, + "f1-score": 0.21410153818796124, + "support": 9533.0 + }, + "accuracy": 0.6658854658854659, + "macro avg": { + "precision": 0.6829013692708266, + "recall": 0.5480799203924813, + "f1-score": 0.5009734342885177, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6774412350755588, + "recall": 0.6658854658854659, + "f1-score": 0.5810980398559723, + "support": 26455.0 + }, + "cross-entropy": 0.6721808608962874, + "roc-auc": 0.6021744792778928, + "score": 0.6021744792778928 + } + }, + "time": "0:03:16.099007", + "diversity": { + "train": { + "mean_centered_corr": -0.01766625978052616, + "mean_pairwise_disagreement": 0.024688098434875683, + "member_std": 0.027821816504001617 + }, + "val": { + "mean_centered_corr": -0.004993404261767864, + "mean_pairwise_disagreement": 0.06782124769476035, + "member_std": 0.05843981355428696 + }, + "test": { + "mean_centered_corr": 0.0002984397578984499, + "mean_pairwise_disagreement": 0.04484334323044, + "member_std": 0.059851765632629395 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 14, + 17, + 11 + ], + "head_selection_time": "0:00:20.397531" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-best-head-evaluation/0/report.json new file mode 100644 index 000000000..96a552803 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-best-head-evaluation/0/report.json @@ -0,0 +1,436 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7887423366275359, + "recall": 0.9689708834745008, + "f1-score": 0.8696166701316147, + "support": 84179.0 + }, + "1": { + "precision": 0.5593048759912266, + "recall": 0.131746284079167, + "f1-score": 0.2132587088680884, + "support": 25162.0 + }, + "accuracy": 0.7763053200537767, + "macro avg": { + "precision": 0.6740236063093812, + "recall": 0.5503585837768339, + "f1-score": 0.5414376894998516, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7359432458516072, + "recall": 0.7763053200537767, + "f1-score": 0.71857288032438, + "support": 109341.0 + }, + "cross-entropy": 0.4989063469933395, + "roc-auc": 0.6823896934462292, + "score": 0.6823896934462292 + }, + "val": { + "0": { + "precision": 0.6937777995014419, + "recall": 0.9146797267689135, + "f1-score": 0.7890596770158712, + "support": 15518.0 + }, + "1": { + "precision": 0.65176223040505, + "recall": 0.28342674139311447, + "f1-score": 0.39505779194898366, + "support": 8743.0 + }, + "accuracy": 0.687193438028111, + "macro avg": { + "precision": 0.672770014953246, + "recall": 0.599053234081014, + "f1-score": 0.5920587344824274, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6786365390171356, + "recall": 0.687193438028111, + "f1-score": 0.6470721875826329, + "support": 24261.0 + }, + "cross-entropy": 0.6253138063351895, + "roc-auc": 0.6475121989956593, + "score": 0.6475121989956593 + }, + "test": { + "0": { + "precision": 0.667395158962517, + "recall": 0.9564472284599929, + "f1-score": 0.786194836421927, + "support": 16922.0 + }, + "1": { + "precision": 0.6656079854809437, + "recall": 0.15388649952795552, + "f1-score": 0.24997869983811877, + "support": 9533.0 + }, + "accuracy": 0.6672462672462672, + "macro avg": { + "precision": 0.6665015722217303, + "recall": 0.5551668639939742, + "f1-score": 0.5180867681300229, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6667511550010792, + "recall": 0.6672462672462672, + "f1-score": 0.5929705525416229, + "support": 26455.0 + }, + "cross-entropy": 0.6598558732426132, + "roc-auc": 0.6043243524106316, + "score": 0.6043243524106316 + } + }, + "time": "0:02:05.402786", + "diversity": { + "train": { + "mean_centered_corr": -0.010739956051111221, + "mean_pairwise_disagreement": 0.02668459002628946, + "member_std": 0.029851045459508896 + }, + "val": { + "mean_centered_corr": -0.013114765286445618, + "mean_pairwise_disagreement": 0.09411593809791634, + "member_std": 0.06746508926153183 + }, + "test": { + "mean_centered_corr": -0.01157404389232397, + "mean_pairwise_disagreement": 0.059169786185915216, + "member_std": 0.06565776467323303 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:04.819667" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0.toml new file mode 100644 index 000000000..0bbba4c1e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0.toml @@ -0,0 +1,67 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0/report.json new file mode 100644 index 000000000..4c4ff62c1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/0/report.json @@ -0,0 +1,432 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7842236626080685, + "recall": 0.9816700127110087, + "f1-score": 0.8719085000422048, + "support": 84179.0 + }, + "1": { + "precision": 0.6111391129032258, + "recall": 0.09637548684524283, + "f1-score": 0.16649502231376587, + "support": 25162.0 + }, + "accuracy": 0.7779424003804611, + "macro avg": { + "precision": 0.6976813877556471, + "recall": 0.5390227497781258, + "f1-score": 0.5192017611779853, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7443927351456048, + "recall": 0.7779424003804611, + "f1-score": 0.7095758533076498, + "support": 109341.0 + }, + "cross-entropy": 0.49281466418323555, + "roc-auc": 0.6980493802481167, + "score": 0.6980493802481167 + }, + "val": { + "0": { + "precision": 0.6769005847953217, + "recall": 0.9547622116252095, + "f1-score": 0.7921723787627654, + "support": 15518.0 + }, + "1": { + "precision": 0.7041719342604298, + "recall": 0.19112432803385565, + "f1-score": 0.3006477150053976, + "support": 8743.0 + }, + "accuracy": 0.6795680309962491, + "macro avg": { + "precision": 0.6905362595278757, + "recall": 0.5729432698295326, + "f1-score": 0.5464100468840815, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6867284323025737, + "recall": 0.6795680309962491, + "f1-score": 0.6150403506010792, + "support": 24261.0 + }, + "cross-entropy": 0.6737994423706947, + "roc-auc": 0.6249365260993432, + "score": 0.6249365260993432 + }, + "test": { + "0": { + "precision": 0.6576644658340626, + "recall": 0.9771303628412717, + "f1-score": 0.7861829593001141, + "support": 16922.0 + }, + "1": { + "precision": 0.7052551408987052, + "recall": 0.09713626350571698, + "f1-score": 0.1707541950949659, + "support": 9533.0 + }, + "accuracy": 0.66002646002646, + "macro avg": { + "precision": 0.6814598033663839, + "recall": 0.5371333131734943, + "f1-score": 0.47846857719754, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6748136590070446, + "recall": 0.66002646002646, + "f1-score": 0.564414582465199, + "support": 26455.0 + }, + "cross-entropy": 0.7355041534289266, + "roc-auc": 0.5925964470819166, + "score": 0.5925964470819166 + } + }, + "time": "0:02:05.402786", + "diversity": { + "train": { + "mean_centered_corr": -0.010739956051111221, + "mean_pairwise_disagreement": 0.02668459002628946, + "member_std": 0.029851045459508896 + }, + "val": { + "mean_centered_corr": -0.013114765286445618, + "mean_pairwise_disagreement": 0.09411593809791634, + "member_std": 0.06746508926153183 + }, + "test": { + "mean_centered_corr": -0.01157404389232397, + "mean_pairwise_disagreement": 0.059169786185915216, + "member_std": 0.06565776467323303 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..08d8e98f7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-greedy-heads-evaluation/0/report.json @@ -0,0 +1,438 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7910060901339829, + "recall": 0.9643378990009385, + "f1-score": 0.8691141517312263, + "support": 84179.0 + }, + "1": { + "precision": 0.5530077427039904, + "recall": 0.14760352913122962, + "f1-score": 0.23301336344814605, + "support": 25162.0 + }, + "accuracy": 0.7763876313551183, + "macro avg": { + "precision": 0.6720069164189867, + "recall": 0.555970714066084, + "f1-score": 0.5510637575896862, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7362369329282369, + "recall": 0.7763876313551183, + "f1-score": 0.7227320257695206, + "support": 109341.0 + }, + "cross-entropy": 0.5000639027992034, + "roc-auc": 0.6809037269331402, + "score": 0.6809037269331402 + }, + "val": { + "0": { + "precision": 0.7070610178523847, + "recall": 0.855007088542338, + "f1-score": 0.7740279438788904, + "support": 15518.0 + }, + "1": { + "precision": 0.5906113537117904, + "recall": 0.37126844332608944, + "f1-score": 0.4559308940234567, + "support": 8743.0 + }, + "accuracy": 0.680680928238737, + "macro avg": { + "precision": 0.6488361857820876, + "recall": 0.6131377659342137, + "f1-score": 0.6149794189511736, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6650957479302374, + "recall": 0.680680928238737, + "f1-score": 0.6593944783627922, + "support": 24261.0 + }, + "cross-entropy": 0.620372456282908, + "roc-auc": 0.6504789566191646, + "score": 0.6504789566191646 + }, + "test": { + "0": { + "precision": 0.6800699300699301, + "recall": 0.9195130599219951, + "f1-score": 0.7818702577759912, + "support": 16922.0 + }, + "1": { + "precision": 0.619020979020979, + "recall": 0.23214098395048777, + "f1-score": 0.33765639304241685, + "support": 9533.0 + }, + "accuracy": 0.6718200718200719, + "macro avg": { + "precision": 0.6495454545454545, + "recall": 0.5758270219362414, + "f1-score": 0.559763325409204, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6580710772878605, + "recall": 0.6718200718200719, + "f1-score": 0.6217987865038247, + "support": 26455.0 + }, + "cross-entropy": 0.6442057237970422, + "roc-auc": 0.6063598206681031, + "score": 0.6063598206681031 + } + }, + "time": "0:02:05.402786", + "diversity": { + "train": { + "mean_centered_corr": -0.010739956051111221, + "mean_pairwise_disagreement": 0.02668459002628946, + "member_std": 0.029851045459508896 + }, + "val": { + "mean_centered_corr": -0.013114765286445618, + "mean_pairwise_disagreement": 0.09411593809791634, + "member_std": 0.06746508926153183 + }, + "test": { + "mean_centered_corr": -0.01157404389232397, + "mean_pairwise_disagreement": 0.059169786185915216, + "member_std": 0.06565776467323303 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 24, + 24 + ], + "head_selection_time": "0:00:20.703567" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..72357a786 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-best-head-evaluation/0/report.json @@ -0,0 +1,390 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.784937732846757, + "recall": 0.978628874184773, + "f1-score": 0.8711468302225982, + "support": 84179.0 + }, + "1": { + "precision": 0.5902050113895216, + "recall": 0.10297273666640172, + "f1-score": 0.17535192203573363, + "support": 25162.0 + }, + "accuracy": 0.7771192873670444, + "macro avg": { + "precision": 0.6875713721181393, + "recall": 0.5408008054255873, + "f1-score": 0.5232493761291659, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.740125039188322, + "recall": 0.7771192873670444, + "f1-score": 0.7110276482158681, + "support": 109341.0 + }, + "cross-entropy": 0.4989568858959861, + "roc-auc": 0.6810795630080747, + "score": 0.6810795630080747 + }, + "val": { + "0": { + "precision": 0.6882539682539682, + "recall": 0.9081067147828329, + "f1-score": 0.783041146889673, + "support": 15518.0 + }, + "1": { + "precision": 0.6233491811938722, + "recall": 0.26993022989820425, + "f1-score": 0.3767259956899992, + "support": 8743.0 + }, + "accuracy": 0.6781253864226536, + "macro avg": { + "precision": 0.6558015747239202, + "recall": 0.5890184723405185, + "f1-score": 0.579883571289836, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6648640604485843, + "recall": 0.6781253864226536, + "f1-score": 0.6366162935473232, + "support": 24261.0 + }, + "cross-entropy": 0.6298773246042463, + "roc-auc": 0.6410202011331968, + "score": 0.6410202011331968 + }, + "test": { + "0": { + "precision": 0.6683168316831684, + "recall": 0.9413780876964898, + "f1-score": 0.781687030766966, + "support": 16922.0 + }, + "1": { + "precision": 0.62122947689958, + "recall": 0.17067030315745305, + "f1-score": 0.26777485187623434, + "support": 9533.0 + }, + "accuracy": 0.6636552636552636, + "macro avg": { + "precision": 0.6447731542913742, + "recall": 0.5560241954269715, + "f1-score": 0.5247309413216001, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6513490088462018, + "recall": 0.6636552636552636, + "f1-score": 0.5964999280882533, + "support": 26455.0 + }, + "cross-entropy": 0.6607089123106477, + "roc-auc": 0.6024701571918213, + "score": 0.6024701571918213 + } + }, + "time": "0:01:11.748716", + "diversity": { + "train": { + "mean_centered_corr": -0.014211982488632202, + "mean_pairwise_disagreement": 0.02565772703979353, + "member_std": 0.02677466720342636 + }, + "val": { + "mean_centered_corr": -0.005956840701401234, + "mean_pairwise_disagreement": 0.08571943089333604, + "member_std": 0.0601416639983654 + }, + "test": { + "mean_centered_corr": 0.00013841522741131485, + "mean_pairwise_disagreement": 0.05374807189323318, + "member_std": 0.05852190777659416 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:04.721148" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0.toml new file mode 100644 index 000000000..0e6ac6970 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0/report.json new file mode 100644 index 000000000..0ed76a5a7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0/report.json @@ -0,0 +1,386 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7863951555903647, + "recall": 0.97808241960584, + "f1-score": 0.8718266385001879, + "support": 84179.0 + }, + "1": { + "precision": 0.602627611458109, + "recall": 0.11119942770844925, + "f1-score": 0.18775373259520214, + "support": 25162.0 + }, + "accuracy": 0.7785917450910455, + "macro avg": { + "precision": 0.6945113835242368, + "recall": 0.5446409236571446, + "f1-score": 0.5297901855476951, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441058135735931, + "recall": 0.7785917450910455, + "f1-score": 0.7144049718025973, + "support": 109341.0 + }, + "cross-entropy": 0.49233822908120295, + "roc-auc": 0.6988844522375441, + "score": 0.6988844522375441 + }, + "val": { + "0": { + "precision": 0.6753571753571753, + "recall": 0.9565021265627014, + "f1-score": 0.7917111158523575, + "support": 15518.0 + }, + "1": { + "precision": 0.7043363994743759, + "recall": 0.1839185634221663, + "f1-score": 0.2916742245601306, + "support": 8743.0 + }, + "accuracy": 0.6780841680062653, + "macro avg": { + "precision": 0.6898467874157757, + "recall": 0.5702103449924338, + "f1-score": 0.5416926702062441, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6858004941180131, + "recall": 0.6780841680062653, + "f1-score": 0.6115115139988503, + "support": 24261.0 + }, + "cross-entropy": 0.6866508172905287, + "roc-auc": 0.6228186386127663, + "score": 0.6228186386127663 + }, + "test": { + "0": { + "precision": 0.6564383127180463, + "recall": 0.97848954024347, + "f1-score": 0.7857447919138234, + "support": 16922.0 + }, + "1": { + "precision": 0.7043054427294883, + "recall": 0.09094723591733976, + "f1-score": 0.16109253065774806, + "support": 9533.0 + }, + "accuracy": 0.6586656586656586, + "macro avg": { + "precision": 0.6803718777237673, + "recall": 0.5347183880804048, + "f1-score": 0.47341866128578575, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6736871258119445, + "recall": 0.6586656586656586, + "f1-score": 0.5606527485740326, + "support": 26455.0 + }, + "cross-entropy": 0.7554621625639649, + "roc-auc": 0.5911736900637132, + "score": 0.5911736900637132 + } + }, + "time": "0:01:11.748716", + "diversity": { + "train": { + "mean_centered_corr": -0.014211982488632202, + "mean_pairwise_disagreement": 0.02565772703979353, + "member_std": 0.02677466720342636 + }, + "val": { + "mean_centered_corr": -0.005956840701401234, + "mean_pairwise_disagreement": 0.08571943089333604, + "member_std": 0.0601416639983654 + }, + "test": { + "mean_centered_corr": 0.00013841522741131485, + "mean_pairwise_disagreement": 0.05374807189323318, + "member_std": 0.05852190777659416 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..edf46fa0f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,391 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7827118483860296, + "recall": 0.9837013982109553, + "f1-score": 0.8717719266847043, + "support": 84179.0 + }, + "1": { + "precision": 0.6130851663846588, + "recall": 0.08640012717590016, + "f1-score": 0.15145604012818725, + "support": 25162.0 + }, + "accuracy": 0.7772107443685351, + "macro avg": { + "precision": 0.6978985073853442, + "recall": 0.5350507626934277, + "f1-score": 0.5116139834064457, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436766596414737, + "recall": 0.7772107443685351, + "f1-score": 0.7060098764241883, + "support": 109341.0 + }, + "cross-entropy": 0.4979833881789, + "roc-auc": 0.6847415093580901, + "score": 0.6847415093580901 + }, + "val": { + "0": { + "precision": 0.6773795373502789, + "recall": 0.9548266529191907, + "f1-score": 0.7925224646983312, + "support": 15518.0 + }, + "1": { + "precision": 0.7063259321323837, + "recall": 0.19283998627473406, + "f1-score": 0.30296495956873315, + "support": 8743.0 + }, + "accuracy": 0.6802275256584642, + "macro avg": { + "precision": 0.6918527347413312, + "recall": 0.5738333195969624, + "f1-score": 0.5477437121335322, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6878110253177965, + "recall": 0.6802275256584642, + "f1-score": 0.6160993466344395, + "support": 24261.0 + }, + "cross-entropy": 0.6392793150076335, + "roc-auc": 0.6419149054445072, + "score": 0.6419149054445072 + }, + "test": { + "0": { + "precision": 0.658847933620552, + "recall": 0.9760075641177166, + "f1-score": 0.7866634913074542, + "support": 16922.0 + }, + "1": { + "precision": 0.7072819033886085, + "recall": 0.10290569600335676, + "f1-score": 0.17967032967032967, + "support": 9533.0 + }, + "accuracy": 0.6613872613872613, + "macro avg": { + "precision": 0.6830649185045803, + "recall": 0.5394566300605367, + "f1-score": 0.48316691048889193, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6763010061512224, + "recall": 0.6613872613872613, + "f1-score": 0.5679348649651103, + "support": 26455.0 + }, + "cross-entropy": 0.6803501878150763, + "roc-auc": 0.6041224275423289, + "score": 0.6041224275423289 + } + }, + "time": "0:01:11.748716", + "diversity": { + "train": { + "mean_centered_corr": -0.014211982488632202, + "mean_pairwise_disagreement": 0.02565772703979353, + "member_std": 0.02677466720342636 + }, + "val": { + "mean_centered_corr": -0.005956840701401234, + "mean_pairwise_disagreement": 0.08571943089333604, + "member_std": 0.0601416639983654 + }, + "test": { + "mean_centered_corr": 0.00013841522741131485, + "mean_pairwise_disagreement": 0.05374807189323318, + "member_std": 0.05852190777659416 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 25 + ], + "head_selection_time": "0:00:15.652537" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e14858863 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-best-head-evaluation/0/report.json @@ -0,0 +1,390 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7849017777184993, + "recall": 0.9787239097637178, + "f1-score": 0.8711623357741415, + "support": 84179.0 + }, + "1": { + "precision": 0.5906285714285714, + "recall": 0.10269453938478658, + "f1-score": 0.17496699055422013, + "support": 25162.0 + }, + "accuracy": 0.7771284330671935, + "macro avg": { + "precision": 0.6877651745735354, + "recall": 0.5407092245742522, + "f1-score": 0.5230646631641808, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7401948295776631, + "recall": 0.7771284330671935, + "f1-score": 0.7109510035527089, + "support": 109341.0 + }, + "cross-entropy": 0.49898712269253065, + "roc-auc": 0.681041160411764, + "score": 0.681041160411764 + }, + "val": { + "0": { + "precision": 0.6882634613505326, + "recall": 0.9077200670189457, + "f1-score": 0.7829035126722989, + "support": 15518.0 + }, + "1": { + "precision": 0.6226613965744401, + "recall": 0.27027336154638, + "f1-score": 0.3769341202743659, + "support": 8743.0 + }, + "accuracy": 0.6780017311734883, + "macro avg": { + "precision": 0.6554624289624864, + "recall": 0.5889967142826629, + "f1-score": 0.5799188164733324, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6646222737516135, + "recall": 0.6780017311734883, + "f1-score": 0.6366032613333134, + "support": 24261.0 + }, + "cross-entropy": 0.630002257594146, + "roc-auc": 0.640920668337369, + "score": 0.640920668337369 + }, + "test": { + "0": { + "precision": 0.668389645318541, + "recall": 0.9399007209549699, + "f1-score": 0.7812269757846653, + "support": 16922.0 + }, + "1": { + "precision": 0.6175253854832644, + "recall": 0.17224378474771845, + "f1-score": 0.2693569553805774, + "support": 9533.0 + }, + "accuracy": 0.6632772632772633, + "macro avg": { + "precision": 0.6429575154009026, + "recall": 0.5560722528513442, + "f1-score": 0.5252919655826214, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6500608232051526, + "recall": 0.6632772632772633, + "f1-score": 0.5967757603428898, + "support": 26455.0 + }, + "cross-entropy": 0.6609475697303346, + "roc-auc": 0.6022998594088651, + "score": 0.6022998594088651 + } + }, + "time": "0:01:12.374421", + "diversity": { + "train": { + "mean_centered_corr": -0.01448837574571371, + "mean_pairwise_disagreement": 0.025696983482570507, + "member_std": 0.02678096853196621 + }, + "val": { + "mean_centered_corr": -0.006191454362124205, + "mean_pairwise_disagreement": 0.0848030690435067, + "member_std": 0.060151606798172 + }, + "test": { + "mean_centered_corr": 2.267197305627633e-05, + "mean_pairwise_disagreement": 0.053483014370111145, + "member_std": 0.05869301036000252 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:04.736645" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0.toml new file mode 100644 index 000000000..6d8c4a5f0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0/report.json new file mode 100644 index 000000000..d31884ffd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-evaluation/0/report.json @@ -0,0 +1,386 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7863576019789309, + "recall": 0.97808241960584, + "f1-score": 0.8718035598945373, + "support": 84179.0 + }, + "1": { + "precision": 0.6021992238033635, + "recall": 0.11100071536443844, + "f1-score": 0.1874496644295302, + "support": 25162.0 + }, + "accuracy": 0.7785460165903001, + "macro avg": { + "precision": 0.6942784128911472, + "recall": 0.5445415674851392, + "f1-score": 0.5296266121620338, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.743978319626898, + "recall": 0.7785460165903001, + "f1-score": 0.7143172307253279, + "support": 109341.0 + }, + "cross-entropy": 0.4923443699583362, + "roc-auc": 0.6988661347453451, + "score": 0.6988661347453451 + }, + "val": { + "0": { + "precision": 0.6754186385147434, + "recall": 0.9565021265627014, + "f1-score": 0.791753347202219, + "support": 15518.0 + }, + "1": { + "precision": 0.7045951859956237, + "recall": 0.18414731785428343, + "f1-score": 0.2919840406238665, + "support": 8743.0 + }, + "accuracy": 0.6781666048390421, + "macro avg": { + "precision": 0.6900069122551835, + "recall": 0.5703247222084924, + "f1-score": 0.5418686939130428, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6859330672120492, + "recall": 0.6781666048390421, + "f1-score": 0.6116501755516467, + "support": 24261.0 + }, + "cross-entropy": 0.6863324565732282, + "roc-auc": 0.6228570653182646, + "score": 0.6228570653182646 + }, + "test": { + "0": { + "precision": 0.6563998890078091, + "recall": 0.9785486349131308, + "f1-score": 0.7857363164013381, + "support": 16922.0 + }, + "1": { + "precision": 0.7043973941368078, + "recall": 0.09073743837197105, + "f1-score": 0.16076572809218473, + "support": 9533.0 + }, + "accuracy": 0.6586278586278587, + "macro avg": { + "precision": 0.6803986415723084, + "recall": 0.534643036642551, + "f1-score": 0.4732510222467614, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6736956824833239, + "recall": 0.6586278586278587, + "f1-score": 0.5605295646209125, + "support": 26455.0 + }, + "cross-entropy": 0.7549517436640245, + "roc-auc": 0.5912083608375949, + "score": 0.5912083608375949 + } + }, + "time": "0:01:12.374421", + "diversity": { + "train": { + "mean_centered_corr": -0.01448837574571371, + "mean_pairwise_disagreement": 0.025696983482570507, + "member_std": 0.02678096853196621 + }, + "val": { + "mean_centered_corr": -0.006191454362124205, + "mean_pairwise_disagreement": 0.0848030690435067, + "member_std": 0.060151606798172 + }, + "test": { + "mean_centered_corr": 2.267197305627633e-05, + "mean_pairwise_disagreement": 0.053483014370111145, + "member_std": 0.05869301036000252 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..88e592e50 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json @@ -0,0 +1,391 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7827204839776917, + "recall": 0.9836657598688509, + "f1-score": 0.871763287694308, + "support": 84179.0 + }, + "1": { + "precision": 0.6127851309490284, + "recall": 0.0864796121135045, + "f1-score": 0.15156897572528122, + "support": 25162.0 + }, + "accuracy": 0.777201598668386, + "macro avg": { + "precision": 0.6977528074633601, + "recall": 0.5350726859911776, + "f1-score": 0.5116661317097946, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436142625885768, + "recall": 0.777201598668386, + "f1-score": 0.7060292146771904, + "support": 109341.0 + }, + "cross-entropy": 0.49797948962611205, + "roc-auc": 0.6847429264219673, + "score": 0.6847429264219673 + }, + "val": { + "0": { + "precision": 0.6772866480778901, + "recall": 0.9548266529191907, + "f1-score": 0.7924588848776575, + "support": 15518.0 + }, + "1": { + "precision": 0.7059563758389261, + "recall": 0.19249685462655838, + "f1-score": 0.3025074143974117, + "support": 8743.0 + }, + "accuracy": 0.6801038704092989, + "macro avg": { + "precision": 0.6916215119584082, + "recall": 0.5736617537728745, + "f1-score": 0.5474831496375345, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6876184328276835, + "recall": 0.6801038704092989, + "f1-score": 0.6158937924903367, + "support": 24261.0 + }, + "cross-entropy": 0.6393924934998982, + "roc-auc": 0.6418041361448852, + "score": 0.6418041361448852 + }, + "test": { + "0": { + "precision": 0.6587311081867847, + "recall": 0.9761848481266989, + "f1-score": 0.7866377770899308, + "support": 16922.0 + }, + "1": { + "precision": 0.7075471698113207, + "recall": 0.1022763033672506, + "f1-score": 0.17871872422326093, + "support": 9533.0 + }, + "accuracy": 0.6612738612738612, + "macro avg": { + "precision": 0.6831391389990527, + "recall": 0.5392305757469747, + "f1-score": 0.48267825065659586, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6763218666621845, + "recall": 0.6612738612738612, + "f1-score": 0.5675755079166946, + "support": 26455.0 + }, + "cross-entropy": 0.6805522467825859, + "roc-auc": 0.6039617691395597, + "score": 0.6039617691395597 + } + }, + "time": "0:01:12.374421", + "diversity": { + "train": { + "mean_centered_corr": -0.01448837574571371, + "mean_pairwise_disagreement": 0.025696983482570507, + "member_std": 0.02678096853196621 + }, + "val": { + "mean_centered_corr": -0.006191454362124205, + "mean_pairwise_disagreement": 0.0848030690435067, + "member_std": 0.060151606798172 + }, + "test": { + "mean_centered_corr": 2.267197305627633e-05, + "mean_pairwise_disagreement": 0.053483014370111145, + "member_std": 0.05869301036000252 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 25 + ], + "head_selection_time": "0:00:15.620571" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5bc424be9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-best-head-evaluation/0/report.json @@ -0,0 +1,390 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7802069833399072, + "recall": 0.9869207284477126, + "f1-score": 0.8714734528823409, + "support": 84179.0 + }, + "1": { + "precision": 0.614900314795383, + "recall": 0.06986726015420078, + "f1-score": 0.12547732058099284, + "support": 25162.0 + }, + "accuracy": 0.7758846178469193, + "macro avg": { + "precision": 0.697553649067645, + "recall": 0.5283939943009567, + "f1-score": 0.49847538673166686, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7421659338349884, + "recall": 0.7758846178469193, + "f1-score": 0.6998017589983769, + "support": 109341.0 + }, + "cross-entropy": 0.49962618774719303, + "roc-auc": 0.6840523798402088, + "score": 0.6840523798402088 + }, + "val": { + "0": { + "precision": 0.6743785937429257, + "recall": 0.9598530738497228, + "f1-score": 0.7921818907060232, + "support": 15518.0 + }, + "1": { + "precision": 0.7134314627414904, + "recall": 0.17739906210682832, + "f1-score": 0.2841439956031877, + "support": 8743.0 + }, + "accuracy": 0.677878075924323, + "macro avg": { + "precision": 0.6939050282422081, + "recall": 0.5686260679782755, + "f1-score": 0.5381629431546054, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6884521782470455, + "recall": 0.677878075924323, + "f1-score": 0.6090989461907892, + "support": 24261.0 + }, + "cross-entropy": 0.6483381636613907, + "roc-auc": 0.638077626500147, + "score": 0.638077626500147 + }, + "test": { + "0": { + "precision": 0.6575799721835883, + "recall": 0.977898593546862, + "f1-score": 0.7863710884596193, + "support": 16922.0 + }, + "1": { + "precision": 0.710077519379845, + "recall": 0.09608727577887338, + "f1-score": 0.16926914903446363, + "support": 9533.0 + }, + "accuracy": 0.6601398601398601, + "macro avg": { + "precision": 0.6838287457817167, + "recall": 0.5369929346628677, + "f1-score": 0.4778201187470415, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6764973457395103, + "recall": 0.6601398601398601, + "f1-score": 0.5639997866815052, + "support": 26455.0 + }, + "cross-entropy": 0.6867257666439333, + "roc-auc": 0.6005513130366958, + "score": 0.6005513130366958 + } + }, + "time": "0:01:07.664836", + "diversity": { + "train": { + "mean_centered_corr": -0.017361776903271675, + "mean_pairwise_disagreement": 0.024792702380330725, + "member_std": 0.027867170050740242 + }, + "val": { + "mean_centered_corr": -0.004654714372009039, + "mean_pairwise_disagreement": 0.06891403433892973, + "member_std": 0.05895354598760605 + }, + "test": { + "mean_centered_corr": 0.0006982876220718026, + "mean_pairwise_disagreement": 0.0455133031745935, + "member_std": 0.06036552041769028 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.884249" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0.toml new file mode 100644 index 000000000..4a734c33a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.2 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0/report.json new file mode 100644 index 000000000..5b73160af --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-evaluation/0/report.json @@ -0,0 +1,386 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.781663133097762, + "recall": 0.9854476769740672, + "f1-score": 0.8718051118210862, + "support": 84179.0 + }, + "1": { + "precision": 0.619092039800995, + "recall": 0.07912725538510453, + "f1-score": 0.14031996617097753, + "support": 25162.0 + }, + "accuracy": 0.7768814991631684, + "macro avg": { + "precision": 0.7003775864493785, + "recall": 0.5322874661795859, + "f1-score": 0.5060625389960318, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7442516054042778, + "recall": 0.7768814991631684, + "f1-score": 0.703472745784119, + "support": 109341.0 + }, + "cross-entropy": 0.49560710251264434, + "roc-auc": 0.6917690308083511, + "score": 0.6917690308083511 + }, + "val": { + "0": { + "precision": 0.6751520101642617, + "recall": 0.958822013146024, + "f1-score": 0.7923634039833848, + "support": 15518.0 + }, + "1": { + "precision": 0.7125506072874493, + "recall": 0.18117351023676084, + "f1-score": 0.28889294182017144, + "support": 8743.0 + }, + "accuracy": 0.6785787890029265, + "macro avg": { + "precision": 0.6938513087258555, + "recall": 0.5699977616913924, + "f1-score": 0.5406281729017781, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6886294403875842, + "recall": 0.6785787890029265, + "f1-score": 0.6109264372180835, + "support": 24261.0 + }, + "cross-entropy": 0.6738878510736607, + "roc-auc": 0.622913421046708, + "score": 0.622913421046708 + }, + "test": { + "0": { + "precision": 0.6571337908056007, + "recall": 0.9790213922704172, + "f1-score": 0.7864144494814042, + "support": 16922.0 + }, + "1": { + "precision": 0.7146302250803859, + "recall": 0.09325500891639568, + "f1-score": 0.16498097800872227, + "support": 9533.0 + }, + "accuracy": 0.6598374598374598, + "macro avg": { + "precision": 0.6858820079429933, + "recall": 0.5361382005934064, + "f1-score": 0.4756977137450632, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6778525021245018, + "recall": 0.6598374598374598, + "f1-score": 0.5624822898310894, + "support": 26455.0 + }, + "cross-entropy": 0.7343241435689584, + "roc-auc": 0.5913362546461658, + "score": 0.5913362546461658 + } + }, + "time": "0:01:07.664836", + "diversity": { + "train": { + "mean_centered_corr": -0.017361776903271675, + "mean_pairwise_disagreement": 0.024792702380330725, + "member_std": 0.027867170050740242 + }, + "val": { + "mean_centered_corr": -0.004654714372009039, + "mean_pairwise_disagreement": 0.06891403433892973, + "member_std": 0.05895354598760605 + }, + "test": { + "mean_centered_corr": 0.0006982876220718026, + "mean_pairwise_disagreement": 0.0455133031745935, + "member_std": 0.06036552041769028 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..56666b5b1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json @@ -0,0 +1,392 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7804729171362031, + "recall": 0.9873127502108602, + "f1-score": 0.8717921821814997, + "support": 84179.0 + }, + "1": { + "precision": 0.6256572029442692, + "recall": 0.07094030681185916, + "f1-score": 0.1274317330001785, + "support": 25162.0 + }, + "accuracy": 0.7764333598558638, + "macro avg": { + "precision": 0.7030650600402362, + "recall": 0.5291265285113597, + "f1-score": 0.4996119575908391, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7448460891348364, + "recall": 0.7764333598558638, + "f1-score": 0.7004968984151138, + "support": 109341.0 + }, + "cross-entropy": 0.4989856299511063, + "roc-auc": 0.6835209664866834, + "score": 0.6835209664866834 + }, + "val": { + "0": { + "precision": 0.678697546434304, + "recall": 0.9536667096275293, + "f1-score": 0.7930230688851378, + "support": 15518.0 + }, + "1": { + "precision": 0.7072475570032574, + "recall": 0.1986732242937207, + "f1-score": 0.3102062684168229, + "support": 8743.0 + }, + "accuracy": 0.6815877333992828, + "macro avg": { + "precision": 0.6929725517187807, + "recall": 0.5761699669606251, + "f1-score": 0.5516146686509804, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6889861883865879, + "recall": 0.6815877333992828, + "f1-score": 0.6190291161834983, + "support": 24261.0 + }, + "cross-entropy": 0.6383314262513924, + "roc-auc": 0.6387008784019832, + "score": 0.6387008784019832 + }, + "test": { + "0": { + "precision": 0.661681942434475, + "recall": 0.972698262616712, + "f1-score": 0.7875974927030002, + "support": 16922.0 + }, + "1": { + "precision": 0.707409753008233, + "recall": 0.11717192908842966, + "f1-score": 0.20104391648668107, + "support": 9533.0 + }, + "accuracy": 0.6644112644112644, + "macro avg": { + "precision": 0.684545847721354, + "recall": 0.5449350958525708, + "f1-score": 0.4943207045948406, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6781598565603354, + "recall": 0.6644112644112644, + "f1-score": 0.5762342251894802, + "support": 26455.0 + }, + "cross-entropy": 0.676206711376993, + "roc-auc": 0.6023730040175572, + "score": 0.6023730040175572 + } + }, + "time": "0:01:07.664836", + "diversity": { + "train": { + "mean_centered_corr": -0.017361776903271675, + "mean_pairwise_disagreement": 0.024792702380330725, + "member_std": 0.027867170050740242 + }, + "val": { + "mean_centered_corr": -0.004654714372009039, + "mean_pairwise_disagreement": 0.06891403433892973, + "member_std": 0.05895354598760605 + }, + "test": { + "mean_centered_corr": 0.0006982876220718026, + "mean_pairwise_disagreement": 0.0455133031745935, + "member_std": 0.06036552041769028 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 11, + 17, + 0 + ], + "head_selection_time": "0:00:20.704412" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a4dec2eba --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,400 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7849948064076541, + "recall": 0.9785694769479324, + "f1-score": 0.8711584424374458, + "support": 84179.0 + }, + "1": { + "precision": 0.5903723887375113, + "recall": 0.10333041888562118, + "f1-score": 0.17587769735507, + "support": 25162.0 + }, + "accuracy": 0.7771558701676406, + "macro avg": { + "precision": 0.6876835975725827, + "recall": 0.5409499479167768, + "f1-score": 0.5235180698962579, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.740207496309739, + "recall": 0.7771558701676406, + "f1-score": 0.7111575817560661, + "support": 109341.0 + }, + "cross-entropy": 0.49905380562363294, + "roc-auc": 0.6808058298435642, + "score": 0.6808058298435642 + }, + "val": { + "0": { + "precision": 0.6883192826860289, + "recall": 0.9102332774842119, + "f1-score": 0.783873026443575, + "support": 15518.0 + }, + "1": { + "precision": 0.6275401069518717, + "recall": 0.268443326089443, + "f1-score": 0.3760314027076824, + "support": 8743.0 + }, + "accuracy": 0.6789497547504225, + "macro avg": { + "precision": 0.6579296948189504, + "recall": 0.5893383017868274, + "f1-score": 0.5799522145756287, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.666416132220519, + "recall": 0.6789497547504225, + "f1-score": 0.6368980742024098, + "support": 24261.0 + }, + "cross-entropy": 0.6300629395883726, + "roc-auc": 0.6410485079831951, + "score": 0.6410485079831951 + }, + "test": { + "0": { + "precision": 0.6681612470667113, + "recall": 0.9422645077414017, + "f1-score": 0.781885941254352, + "support": 16922.0 + }, + "1": { + "precision": 0.6229255113855654, + "recall": 0.16930661911255637, + "f1-score": 0.26624876278455956, + "support": 9533.0 + }, + "accuracy": 0.6637308637308638, + "macro avg": { + "precision": 0.6455433792261384, + "recall": 0.555785563426979, + "f1-score": 0.5240673520194558, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6518606510263272, + "recall": 0.6637308637308638, + "f1-score": 0.5960772388407238, + "support": 26455.0 + }, + "cross-entropy": 0.6611287604378554, + "roc-auc": 0.6024310820580536, + "score": 0.6024310820580536 + } + }, + "time": "0:01:57.344730", + "diversity": { + "train": { + "mean_centered_corr": -0.014402652159333229, + "mean_pairwise_disagreement": 0.025755084492993362, + "member_std": 0.02682984620332718 + }, + "val": { + "mean_centered_corr": -0.0061649116687476635, + "mean_pairwise_disagreement": 0.08504215247888885, + "member_std": 0.060025766491889954 + }, + "test": { + "mean_centered_corr": -0.0001455514138797298, + "mean_pairwise_disagreement": 0.05350869705708415, + "member_std": 0.05860954150557518 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:04.814924" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..025615a45 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0.toml @@ -0,0 +1,54 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..97857fc20 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/0/report.json @@ -0,0 +1,396 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7863774428334002, + "recall": 0.9780230223689994, + "f1-score": 0.8717921566343522, + "support": 84179.0 + }, + "1": { + "precision": 0.6018936948568969, + "recall": 0.11115968523964709, + "f1-score": 0.18766144453017544, + "support": 25162.0 + }, + "accuracy": 0.778536870890151, + "macro avg": { + "precision": 0.6941355688451485, + "recall": 0.5445913538043232, + "f1-score": 0.5297268005822638, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.743923285046433, + "recall": 0.778536870890151, + "f1-score": 0.7143571873367851, + "support": 109341.0 + }, + "cross-entropy": 0.49238150256808705, + "roc-auc": 0.6987779918614105, + "score": 0.6987779918614105 + }, + "val": { + "0": { + "precision": 0.6755440225803514, + "recall": 0.9562443613867766, + "f1-score": 0.7917511471561199, + "support": 15518.0 + }, + "1": { + "precision": 0.7041394335511982, + "recall": 0.1848335811506348, + "f1-score": 0.29280666787461496, + "support": 8743.0 + }, + "accuracy": 0.678249041671819, + "macro avg": { + "precision": 0.6898417280657748, + "recall": 0.5705389712687057, + "f1-score": 0.5422789075153674, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6858490255941642, + "recall": 0.678249041671819, + "f1-score": 0.6119452206750104, + "support": 24261.0 + }, + "cross-entropy": 0.686338046017919, + "roc-auc": 0.6227702320934685, + "score": 0.6227702320934685 + }, + "test": { + "0": { + "precision": 0.6565536387071188, + "recall": 0.9783122562344877, + "f1-score": 0.7857702256923844, + "support": 16922.0 + }, + "1": { + "precision": 0.7040322580645161, + "recall": 0.09157662855344592, + "f1-score": 0.1620718462823726, + "support": 9533.0 + }, + "accuracy": 0.6587790587790587, + "macro avg": { + "precision": 0.6802929483858174, + "recall": 0.5349444423939668, + "f1-score": 0.4739210359873785, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6736624528569607, + "recall": 0.6587790587790587, + "f1-score": 0.5610219115394589, + "support": 26455.0 + }, + "cross-entropy": 0.754977849360994, + "roc-auc": 0.5911624823470715, + "score": 0.5911624823470715 + } + }, + "time": "0:01:57.344730", + "diversity": { + "train": { + "mean_centered_corr": -0.014402652159333229, + "mean_pairwise_disagreement": 0.025755084492993362, + "member_std": 0.02682984620332718 + }, + "val": { + "mean_centered_corr": -0.0061649116687476635, + "mean_pairwise_disagreement": 0.08504215247888885, + "member_std": 0.060025766491889954 + }, + "test": { + "mean_centered_corr": -0.0001455514138797298, + "mean_pairwise_disagreement": 0.05350869705708415, + "member_std": 0.05860954150557518 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..a5b8aaea0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,401 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7827360978514647, + "recall": 0.9837132776583233, + "f1-score": 0.871791632451098, + "support": 84179.0 + }, + "1": { + "precision": 0.6135851183765502, + "recall": 0.08651935458230665, + "f1-score": 0.15165447579240682, + "support": 25162.0 + }, + "accuracy": 0.7772473271691315, + "macro avg": { + "precision": 0.6981606081140075, + "recall": 0.535116316120315, + "f1-score": 0.5117230541217525, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.743810379726079, + "recall": 0.7772473271691315, + "f1-score": 0.7060707122487404, + "support": 109341.0 + }, + "cross-entropy": 0.49803582395239526, + "roc-auc": 0.6845751097530018, + "score": 0.6845751097530018 + }, + "val": { + "0": { + "precision": 0.6770657287717535, + "recall": 0.9552133006830777, + "f1-score": 0.7924407259896822, + "support": 15518.0 + }, + "1": { + "precision": 0.7065033783783784, + "recall": 0.19135308246597277, + "f1-score": 0.3011430114301143, + "support": 8743.0 + }, + "accuracy": 0.6799389967437451, + "macro avg": { + "precision": 0.691784553575066, + "recall": 0.5732831915745252, + "f1-score": 0.5467918687098983, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6876742515247612, + "recall": 0.6799389967437451, + "f1-score": 0.6153904841037623, + "support": 24261.0 + }, + "cross-entropy": 0.6393443379299147, + "roc-auc": 0.6417659932080955, + "score": 0.6417659932080955 + }, + "test": { + "0": { + "precision": 0.6586659224113871, + "recall": 0.9762439427963597, + "f1-score": 0.786610480203795, + "support": 16922.0 + }, + "1": { + "precision": 0.7074235807860262, + "recall": 0.10196160704919753, + "f1-score": 0.17823416154762997, + "support": 9533.0 + }, + "accuracy": 0.6611982611982612, + "macro avg": { + "precision": 0.6830447515987066, + "recall": 0.5391027749227786, + "f1-score": 0.48242232087571246, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6762356354064896, + "recall": 0.6611982611982612, + "f1-score": 0.5673834363274305, + "support": 26455.0 + }, + "cross-entropy": 0.6804458567708631, + "roc-auc": 0.6040036864957168, + "score": 0.6040036864957168 + } + }, + "time": "0:01:57.344730", + "diversity": { + "train": { + "mean_centered_corr": -0.014402652159333229, + "mean_pairwise_disagreement": 0.025755084492993362, + "member_std": 0.02682984620332718 + }, + "val": { + "mean_centered_corr": -0.0061649116687476635, + "mean_pairwise_disagreement": 0.08504215247888885, + "member_std": 0.060025766491889954 + }, + "test": { + "mean_centered_corr": -0.0001455514138797298, + "mean_pairwise_disagreement": 0.05350869705708415, + "member_std": 0.05860954150557518 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 25 + ], + "head_selection_time": "0:00:15.908281" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4541709e6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,265 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7919478842655086, + "recall": 0.9618075767115314, + "f1-score": 0.8686518644085981, + "support": 84179.0 + }, + "1": { + "precision": 0.5476290980723231, + "recall": 0.15467768857801448, + "f1-score": 0.2412222256654994, + "support": 25162.0 + }, + "accuracy": 0.7760675318499007, + "macro avg": { + "precision": 0.6697884911689158, + "recall": 0.5582426326447729, + "f1-score": 0.5549370450370488, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7357242417325802, + "recall": 0.7760675318499007, + "f1-score": 0.7242651789927537, + "support": 109341.0 + }, + "cross-entropy": 0.501967718593187, + "roc-auc": 0.6776997459791548, + "score": 0.6776997459791548 + }, + "val": { + "0": { + "precision": 0.7099598084016957, + "recall": 0.8309704858873567, + "f1-score": 0.7657136070781746, + "support": 15518.0 + }, + "1": { + "precision": 0.5698589701541489, + "recall": 0.39746082580349995, + "f1-score": 0.46829728454955866, + "support": 8743.0 + }, + "accuracy": 0.6747454762788013, + "macro avg": { + "precision": 0.6399093892779223, + "recall": 0.6142156558454284, + "f1-score": 0.6170054458138666, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6594713030310061, + "recall": 0.6747454762788013, + "f1-score": 0.6585329093382757, + "support": 24261.0 + }, + "cross-entropy": 0.6236491778893606, + "roc-auc": 0.6500013075472437, + "score": 0.6500013075472437 + }, + "test": { + "0": { + "precision": 0.6826944581115534, + "recall": 0.9019619430327385, + "f1-score": 0.7771582779602332, + "support": 16922.0 + }, + "1": { + "precision": 0.5951683748169839, + "recall": 0.25584810657715307, + "f1-score": 0.3578607585650356, + "support": 9533.0 + }, + "accuracy": 0.6691362691362691, + "macro avg": { + "precision": 0.6389314164642687, + "recall": 0.5789050248049458, + "f1-score": 0.5675095182626344, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6511546300243438, + "recall": 0.6691362691362691, + "f1-score": 0.6260653559268022, + "support": 26455.0 + }, + "cross-entropy": 0.6442269054009311, + "roc-auc": 0.6046018053871005, + "score": 0.6046018053871005 + } + }, + "time": "0:01:42.652042", + "diversity": { + "train": { + "mean_centered_corr": -0.011058990843594074, + "mean_pairwise_disagreement": 0.026724399636414167, + "member_std": 0.03006140887737274 + }, + "val": { + "mean_centered_corr": -0.00816602073609829, + "mean_pairwise_disagreement": 0.09307434206764872, + "member_std": 0.06737758219242096 + }, + "test": { + "mean_centered_corr": -0.005533241201192141, + "mean_pairwise_disagreement": 0.05856963437608599, + "member_std": 0.06563688069581985 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24 + ], + "head_selection_time": "0:00:04.775982" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0.toml new file mode 100644 index 000000000..aaf475ea7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0.toml @@ -0,0 +1,56 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0/report.json new file mode 100644 index 000000000..5cdd60541 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/0/report.json @@ -0,0 +1,261 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7842988383569964, + "recall": 0.9817175305004812, + "f1-score": 0.8719737057299772, + "support": 84179.0 + }, + "1": { + "precision": 0.6126352881953184, + "recall": 0.09673316906446229, + "f1-score": 0.16708426291402093, + "support": 25162.0 + }, + "accuracy": 0.7780612944823991, + "macro avg": { + "precision": 0.6984670632761574, + "recall": 0.5392253497824717, + "f1-score": 0.5195289843219991, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7447949171456655, + "recall": 0.7780612944823991, + "f1-score": 0.7097616520617732, + "support": 109341.0 + }, + "cross-entropy": 0.49277000777319097, + "roc-auc": 0.6981352050298899, + "score": 0.6981352050298899 + }, + "val": { + "0": { + "precision": 0.6762921399571188, + "recall": 0.95534218327104, + "f1-score": 0.7919549133256765, + "support": 15518.0 + }, + "1": { + "precision": 0.7038461538461539, + "recall": 0.18837927484845018, + "f1-score": 0.2972119462239466, + "support": 8743.0 + }, + "accuracy": 0.6789497547504225, + "macro avg": { + "precision": 0.6900691469016363, + "recall": 0.571860729059745, + "f1-score": 0.5445834297748116, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6862218519818429, + "recall": 0.6789497547504225, + "f1-score": 0.6136630967735796, + "support": 24261.0 + }, + "cross-entropy": 0.6746455512551084, + "roc-auc": 0.6249124647240485, + "score": 0.6249124647240485 + }, + "test": { + "0": { + "precision": 0.6572768087726966, + "recall": 0.9776031201985581, + "f1-score": 0.786058777410848, + "support": 16922.0 + }, + "1": { + "precision": 0.7052877138413686, + "recall": 0.09514318682471415, + "f1-score": 0.16766799149644143, + "support": 9533.0 + }, + "accuracy": 0.6596106596106596, + "macro avg": { + "precision": 0.6812822613070326, + "recall": 0.5363731535116362, + "f1-score": 0.4768633844536447, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6745774308864615, + "recall": 0.6596106596106596, + "f1-score": 0.5632230426869002, + "support": 26455.0 + }, + "cross-entropy": 0.7368636466213944, + "roc-auc": 0.5926966501436739, + "score": 0.5926966501436739 + } + }, + "time": "0:01:42.652042", + "diversity": { + "train": { + "mean_centered_corr": -0.011058990843594074, + "mean_pairwise_disagreement": 0.026724399636414167, + "member_std": 0.03006140887737274 + }, + "val": { + "mean_centered_corr": -0.00816602073609829, + "mean_pairwise_disagreement": 0.09307434206764872, + "member_std": 0.06737758219242096 + }, + "test": { + "mean_centered_corr": -0.005533241201192141, + "mean_pairwise_disagreement": 0.05856963437608599, + "member_std": 0.06563688069581985 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7e5a636a1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,266 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7903039369007956, + "recall": 0.9665237173166704, + "f1-score": 0.8695759052627079, + "support": 84179.0 + }, + "1": { + "precision": 0.559136420525657, + "recall": 0.14203958349892695, + "f1-score": 0.22653229384547124, + "support": 25162.0 + }, + "accuracy": 0.7767900421616777, + "macro avg": { + "precision": 0.6747201787132263, + "recall": 0.5542816504077986, + "f1-score": 0.5480540995540896, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7371067185926472, + "recall": 0.7767900421616777, + "f1-score": 0.7215960683261471, + "support": 109341.0 + }, + "cross-entropy": 0.49894859408324227, + "roc-auc": 0.6833477924522856, + "score": 0.6833477924522856 + }, + "val": { + "0": { + "precision": 0.7034465020576132, + "recall": 0.8812346951926795, + "f1-score": 0.7823674123233595, + "support": 15518.0 + }, + "1": { + "precision": 0.6177141671852313, + "recall": 0.3406153494223951, + "f1-score": 0.4391035092892952, + "support": 8743.0 + }, + "accuracy": 0.6864102881167305, + "macro avg": { + "precision": 0.6605803346214223, + "recall": 0.6109250223075373, + "f1-score": 0.6107354608063273, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6725509163938221, + "recall": 0.6864102881167305, + "f1-score": 0.6586645021289395, + "support": 24261.0 + }, + "cross-entropy": 0.6204343840179957, + "roc-auc": 0.650368091501537, + "score": 0.650368091501537 + }, + "test": { + "0": { + "precision": 0.676635434047833, + "recall": 0.9345822006854981, + "f1-score": 0.784960913264673, + "support": 16922.0 + }, + "1": { + "precision": 0.6408176508760545, + "recall": 0.2071750760516102, + "f1-score": 0.31311930241775665, + "support": 9533.0 + }, + "accuracy": 0.6724626724626724, + "macro avg": { + "precision": 0.6587265424619437, + "recall": 0.5708786383685541, + "f1-score": 0.5490401078412148, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6637285761012608, + "recall": 0.6724626724626724, + "f1-score": 0.6149338455571072, + "support": 26455.0 + }, + "cross-entropy": 0.6488325155298535, + "roc-auc": 0.6061843343570334, + "score": 0.6061843343570334 + } + }, + "time": "0:01:42.652042", + "diversity": { + "train": { + "mean_centered_corr": -0.011058990843594074, + "mean_pairwise_disagreement": 0.026724399636414167, + "member_std": 0.03006140887737274 + }, + "val": { + "mean_centered_corr": -0.00816602073609829, + "mean_pairwise_disagreement": 0.09307434206764872, + "member_std": 0.06737758219242096 + }, + "test": { + "mean_centered_corr": -0.005533241201192141, + "mean_pairwise_disagreement": 0.05856963437608599, + "member_std": 0.06563688069581985 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24, + 22 + ], + "head_selection_time": "0:00:15.768684" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..223176d4b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7841790931223371, + "recall": 0.9795554710794854, + "f1-score": 0.8710459459887709, + "support": 84179.0 + }, + "1": { + "precision": 0.5891620911912151, + "recall": 0.0980844130037358, + "f1-score": 0.16817144219958435, + "support": 25162.0 + }, + "accuracy": 0.7767077308603361, + "macro avg": { + "precision": 0.6866705921567762, + "recall": 0.5388199420416105, + "f1-score": 0.5196086940941776, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7393009796736684, + "recall": 0.7767077308603361, + "f1-score": 0.709297578365066, + "support": 109341.0 + }, + "cross-entropy": 0.4992703210863206, + "roc-auc": 0.6803857932728635, + "score": 0.6803857932728635 + }, + "val": { + "0": { + "precision": 0.688133775351014, + "recall": 0.9095888645444, + "f1-score": 0.7835137385512073, + "support": 15518.0 + }, + "1": { + "precision": 0.6257668711656442, + "recall": 0.26832894887338443, + "f1-score": 0.3756003842459174, + "support": 8743.0 + }, + "accuracy": 0.6784963521701496, + "macro avg": { + "precision": 0.6569503232583291, + "recall": 0.5889589067088923, + "f1-score": 0.5795570613985623, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6656584510324497, + "recall": 0.6784963521701496, + "f1-score": 0.63651293657721, + "support": 24261.0 + }, + "cross-entropy": 0.6310932933009203, + "roc-auc": 0.6410241296714208, + "score": 0.6410241296714208 + }, + "test": { + "0": { + "precision": 0.6670702684647822, + "recall": 0.9441555371705472, + "f1-score": 0.781787488072811, + "support": 16922.0 + }, + "1": { + "precision": 0.6226038338658147, + "recall": 0.16353718661491662, + "f1-score": 0.2590346431835175, + "support": 9533.0 + }, + "accuracy": 0.6628614628614629, + "macro avg": { + "precision": 0.6448370511652985, + "recall": 0.553846361892732, + "f1-score": 0.5204110656281642, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6510468883463941, + "recall": 0.6628614628614629, + "f1-score": 0.5934146712015339, + "support": 26455.0 + }, + "cross-entropy": 0.6635058051130894, + "roc-auc": 0.6024966050474919, + "score": 0.6024966050474919 + } + }, + "time": "0:01:33.915789", + "diversity": { + "train": { + "mean_centered_corr": -0.013111491687595844, + "mean_pairwise_disagreement": 0.02585577938919114, + "member_std": 0.02714134193956852 + }, + "val": { + "mean_centered_corr": -0.004636597353965044, + "mean_pairwise_disagreement": 0.08690653790565238, + "member_std": 0.060862503945827484 + }, + "test": { + "mean_centered_corr": 0.0027443149592727423, + "mean_pairwise_disagreement": 0.05465580626870949, + "member_std": 0.05927826836705208 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:04.756597" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0.toml new file mode 100644 index 000000000..9b75584bc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0/report.json new file mode 100644 index 000000000..517e309dc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7862810438170169, + "recall": 0.9782487318689934, + "f1-score": 0.8718225610078874, + "support": 84179.0 + }, + "1": { + "precision": 0.602819956616052, + "recall": 0.11044432080120817, + "f1-score": 0.18668547628644364, + "support": 25162.0 + }, + "accuracy": 0.7785460165903001, + "macro avg": { + "precision": 0.6945505002165344, + "recall": 0.5443465263351008, + "f1-score": 0.5292540186471655, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7440622249279388, + "recall": 0.7785460165903001, + "f1-score": 0.7141560011103103, + "support": 109341.0 + }, + "cross-entropy": 0.4922902286245573, + "roc-auc": 0.6989728271677539, + "score": 0.6989728271677539 + }, + "val": { + "0": { + "precision": 0.675189954047045, + "recall": 0.9563088026807578, + "f1-score": 0.791529989065792, + "support": 15518.0 + }, + "1": { + "precision": 0.7028921998247152, + "recall": 0.18346105455793205, + "f1-score": 0.2909750566893424, + "support": 8743.0 + }, + "accuracy": 0.677795639091546, + "macro avg": { + "precision": 0.6890410769358801, + "recall": 0.5698849286193449, + "f1-score": 0.5412525228775672, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6851730847850266, + "recall": 0.677795639091546, + "f1-score": 0.6111436993923531, + "support": 24261.0 + }, + "cross-entropy": 0.6872600691770208, + "roc-auc": 0.6227874903903754, + "score": 0.6227874903903754 + }, + "test": { + "0": { + "precision": 0.656414727914074, + "recall": 0.9787259189221132, + "f1-score": 0.7858040946077385, + "support": 16922.0 + }, + "1": { + "precision": 0.7058823529411765, + "recall": 0.09063253959928669, + "f1-score": 0.16063958352700566, + "support": 9533.0 + }, + "accuracy": 0.6587034587034587, + "macro avg": { + "precision": 0.6811485404276252, + "recall": 0.5346792292607, + "f1-score": 0.4732218390673721, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6742402758023133, + "recall": 0.6587034587034587, + "f1-score": 0.5605274631908937, + "support": 26455.0 + }, + "cross-entropy": 0.7564828184565525, + "roc-auc": 0.5911993816464689, + "score": 0.5911993816464689 + } + }, + "time": "0:01:33.915789", + "diversity": { + "train": { + "mean_centered_corr": -0.013111491687595844, + "mean_pairwise_disagreement": 0.02585577938919114, + "member_std": 0.02714134193956852 + }, + "val": { + "mean_centered_corr": -0.004636597353965044, + "mean_pairwise_disagreement": 0.08690653790565238, + "member_std": 0.060862503945827484 + }, + "test": { + "mean_centered_corr": 0.0027443149592727423, + "mean_pairwise_disagreement": 0.05465580626870949, + "member_std": 0.05927826836705208 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..a92177d15 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,230 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.782973008757495, + "recall": 0.9834994476056974, + "f1-score": 0.871854546986315, + "support": 84179.0 + }, + "1": { + "precision": 0.6144879267277269, + "recall": 0.08798982592798664, + "f1-score": 0.1539370763080132, + "support": 25162.0 + }, + "accuracy": 0.7774210954719638, + "macro avg": { + "precision": 0.6987304677426109, + "recall": 0.535744636766842, + "f1-score": 0.5128958116471641, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7442005296871277, + "recall": 0.7774210954719638, + "f1-score": 0.7066444300383502, + "support": 109341.0 + }, + "cross-entropy": 0.4979289521091686, + "roc-auc": 0.6847935169951291, + "score": 0.6847935169951291 + }, + "val": { + "0": { + "precision": 0.6776083871263104, + "recall": 0.9537955922154917, + "f1-score": 0.7923235459436311, + "support": 15518.0 + }, + "1": { + "precision": 0.7034739454094293, + "recall": 0.1945556445156125, + "f1-score": 0.3048113968282412, + "support": 8743.0 + }, + "accuracy": 0.6801863072420757, + "macro avg": { + "precision": 0.6905411662678699, + "recall": 0.5741756183655521, + "f1-score": 0.5485674713859361, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6869296259898902, + "recall": 0.6801863072420757, + "f1-score": 0.6166375181741305, + "support": 24261.0 + }, + "cross-entropy": 0.639843807695552, + "roc-auc": 0.641845068859757, + "score": 0.641845068859757 + }, + "test": { + "0": { + "precision": 0.6588995730758489, + "recall": 0.975889374778395, + "f1-score": 0.7866619030606169, + "support": 16922.0 + }, + "1": { + "precision": 0.7068965517241379, + "recall": 0.10322039232140984, + "f1-score": 0.18013729977116705, + "support": 9533.0 + }, + "accuracy": 0.6614250614250614, + "macro avg": { + "precision": 0.6828980623999934, + "recall": 0.5395548835499024, + "f1-score": 0.483399601415892, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6761951768352191, + "recall": 0.6614250614250614, + "f1-score": 0.5681021206694499, + "support": 26455.0 + }, + "cross-entropy": 0.6818687369091303, + "roc-auc": 0.6041622806453656, + "score": 0.6041622806453656 + } + }, + "time": "0:01:33.915789", + "diversity": { + "train": { + "mean_centered_corr": -0.013111491687595844, + "mean_pairwise_disagreement": 0.02585577938919114, + "member_std": 0.02714134193956852 + }, + "val": { + "mean_centered_corr": -0.004636597353965044, + "mean_pairwise_disagreement": 0.08690653790565238, + "member_std": 0.060862503945827484 + }, + "test": { + "mean_centered_corr": 0.0027443149592727423, + "mean_pairwise_disagreement": 0.05465580626870949, + "member_std": 0.05927826836705208 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 25 + ], + "head_selection_time": "0:00:15.827629" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..32626ae07 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7842672905729285, + "recall": 0.9795911094215897, + "f1-score": 0.8711144446263792, + "support": 84179.0 + }, + "1": { + "precision": 0.5906599952346915, + "recall": 0.09852158016055958, + "f1-score": 0.16887496168125618, + "support": 25162.0 + }, + "accuracy": 0.776835770662423, + "macro avg": { + "precision": 0.68746364290381, + "recall": 0.5390563447910747, + "f1-score": 0.5199947031538177, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7397135845952924, + "recall": 0.776835770662423, + "f1-score": 0.7095122106074367, + "support": 109341.0 + }, + "cross-entropy": 0.49927958217372836, + "roc-auc": 0.6803137982602562, + "score": 0.6803137982602562 + }, + "val": { + "0": { + "precision": 0.6883377360336562, + "recall": 0.906753447609228, + "f1-score": 0.7825917686318131, + "support": 15518.0 + }, + "1": { + "precision": 0.6211050013092433, + "recall": 0.271302756490907, + "f1-score": 0.37764687151727433, + "support": 8743.0 + }, + "accuracy": 0.6777544206751577, + "macro avg": { + "precision": 0.6547213686714497, + "recall": 0.5890281020500675, + "f1-score": 0.5801193200745437, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6641088996421002, + "recall": 0.6777544206751577, + "f1-score": 0.636660717336631, + "support": 24261.0 + }, + "cross-entropy": 0.6314923590147058, + "roc-auc": 0.64045684285539, + "score": 0.64045684285539 + }, + "test": { + "0": { + "precision": 0.6676155391828533, + "recall": 0.9424417917503841, + "f1-score": 0.7815731438372948, + "support": 16922.0 + }, + "1": { + "precision": 0.6205687573042462, + "recall": 0.16710374488618482, + "f1-score": 0.26330578512396696, + "support": 9533.0 + }, + "accuracy": 0.663050463050463, + "macro avg": { + "precision": 0.6440921482435498, + "recall": 0.5547727683182845, + "f1-score": 0.5224394644806309, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6506623367013276, + "recall": 0.663050463050463, + "f1-score": 0.5948166618635978, + "support": 26455.0 + }, + "cross-entropy": 0.6635800829766905, + "roc-auc": 0.6022469172053364, + "score": 0.6022469172053364 + } + }, + "time": "0:01:33.840464", + "diversity": { + "train": { + "mean_centered_corr": -0.013077761046588421, + "mean_pairwise_disagreement": 0.02595400347123574, + "member_std": 0.027166401967406273 + }, + "val": { + "mean_centered_corr": -0.004114337265491486, + "mean_pairwise_disagreement": 0.0868821060217447, + "member_std": 0.060897231101989746 + }, + "test": { + "mean_centered_corr": 0.003803197992965579, + "mean_pairwise_disagreement": 0.05452624968754001, + "member_std": 0.05941416695713997 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:04.729629" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0.toml new file mode 100644 index 000000000..361754c34 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0/report.json new file mode 100644 index 000000000..4a2c19b44 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7862332330898849, + "recall": 0.9783200085532021, + "f1-score": 0.8718214731849845, + "support": 84179.0 + }, + "1": { + "precision": 0.6029155787641427, + "recall": 0.11012638105079088, + "f1-score": 0.18623563411519592, + "support": 25162.0 + }, + "accuracy": 0.7785277251900019, + "macro avg": { + "precision": 0.6945744059270138, + "recall": 0.5442231948019965, + "f1-score": 0.5290285536500902, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7440474215631535, + "recall": 0.7785277251900019, + "f1-score": 0.7140516440936644, + "support": 109341.0 + }, + "cross-entropy": 0.4923044781282782, + "roc-auc": 0.6989427314976193, + "score": 0.6989427314976193 + }, + "val": { + "0": { + "precision": 0.6752206752206752, + "recall": 0.9563088026807578, + "f1-score": 0.7915510987838703, + "support": 15518.0 + }, + "1": { + "precision": 0.7030223390275953, + "recall": 0.18357543177399063, + "f1-score": 0.29113005623072735, + "support": 8743.0 + }, + "accuracy": 0.6778368575079345, + "macro avg": { + "precision": 0.6891215071241352, + "recall": 0.5699421172273742, + "f1-score": 0.5413405775072988, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6852396334937845, + "recall": 0.6778368575079345, + "f1-score": 0.6112130593361094, + "support": 24261.0 + }, + "cross-entropy": 0.6870530045533841, + "roc-auc": 0.6227333347907498, + "score": 0.6227333347907498 + }, + "test": { + "0": { + "precision": 0.6564531472966545, + "recall": 0.9786668242524524, + "f1-score": 0.7858125741399763, + "support": 16922.0 + }, + "1": { + "precision": 0.7057864710676447, + "recall": 0.09084233714465541, + "f1-score": 0.16096654275092936, + "support": 9533.0 + }, + "accuracy": 0.6587412587412588, + "macro avg": { + "precision": 0.6811198091821495, + "recall": 0.534754580698554, + "f1-score": 0.47338955844545283, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6742303000280417, + "recall": 0.6587412587412588, + "f1-score": 0.5606507061667468, + "support": 26455.0 + }, + "cross-entropy": 0.7560531851992458, + "roc-auc": 0.5911601949314514, + "score": 0.5911601949314514 + } + }, + "time": "0:01:33.840464", + "diversity": { + "train": { + "mean_centered_corr": -0.013077761046588421, + "mean_pairwise_disagreement": 0.02595400347123574, + "member_std": 0.027166401967406273 + }, + "val": { + "mean_centered_corr": -0.004114337265491486, + "mean_pairwise_disagreement": 0.0868821060217447, + "member_std": 0.060897231101989746 + }, + "test": { + "mean_centered_corr": 0.003803197992965579, + "mean_pairwise_disagreement": 0.05452624968754001, + "member_std": 0.05941416695713997 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..42478e4a6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,230 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7829495484420067, + "recall": 0.9835350859478017, + "f1-score": 0.8718540047597987, + "support": 84179.0 + }, + "1": { + "precision": 0.6145717463848721, + "recall": 0.087830856052778, + "f1-score": 0.15369636275123444, + "support": 25162.0 + }, + "accuracy": 0.7774119497718148, + "macro avg": { + "precision": 0.6987606474134394, + "recall": 0.5356829710002898, + "f1-score": 0.5127751837555166, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7442017570795569, + "recall": 0.7774119497718148, + "f1-score": 0.7065886185988938, + "support": 109341.0 + }, + "cross-entropy": 0.4979388170923032, + "roc-auc": 0.6847377557794279, + "score": 0.6847377557794279 + }, + "val": { + "0": { + "precision": 0.6775373919407218, + "recall": 0.9545688877432659, + "f1-score": 0.7925416655520184, + "support": 15518.0 + }, + "1": { + "precision": 0.7060050041701418, + "recall": 0.193640626787144, + "f1-score": 0.3039224486132304, + "support": 8743.0 + }, + "accuracy": 0.6803511809076296, + "macro avg": { + "precision": 0.6917711980554317, + "recall": 0.5741047572652049, + "f1-score": 0.5482320570826245, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6877963397879588, + "recall": 0.6803511809076296, + "f1-score": 0.6164566808565887, + "support": 24261.0 + }, + "cross-entropy": 0.6400045165065813, + "roc-auc": 0.641480466607742, + "score": 0.641480466607742 + }, + "test": { + "0": { + "precision": 0.6585200542221513, + "recall": 0.9760666587873774, + "f1-score": 0.7864489096276545, + "support": 16922.0 + }, + "1": { + "precision": 0.7050254916241806, + "recall": 0.10154201195846009, + "f1-score": 0.1775169631395562, + "support": 9533.0 + }, + "accuracy": 0.6609336609336609, + "macro avg": { + "precision": 0.681772772923166, + "recall": 0.5388043353729187, + "f1-score": 0.48198293638360534, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6752781844339656, + "recall": 0.6609336609336609, + "f1-score": 0.567021646506466, + "support": 26455.0 + }, + "cross-entropy": 0.6817906405461313, + "roc-auc": 0.6039784195416061, + "score": 0.6039784195416061 + } + }, + "time": "0:01:33.840464", + "diversity": { + "train": { + "mean_centered_corr": -0.013077761046588421, + "mean_pairwise_disagreement": 0.02595400347123574, + "member_std": 0.027166401967406273 + }, + "val": { + "mean_centered_corr": -0.004114337265491486, + "mean_pairwise_disagreement": 0.0868821060217447, + "member_std": 0.060897231101989746 + }, + "test": { + "mean_centered_corr": 0.003803197992965579, + "mean_pairwise_disagreement": 0.05452624968754001, + "member_std": 0.05941416695713997 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 25 + ], + "head_selection_time": "0:00:15.726306" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..aa7be9cc7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7843107416636549, + "recall": 0.9796267477636941, + "f1-score": 0.8711553393442882, + "support": 84179.0 + }, + "1": { + "precision": 0.5915694212907835, + "recall": 0.09872029250457039, + "f1-score": 0.1692040461837131, + "support": 25162.0 + }, + "accuracy": 0.7769089362636157, + "macro avg": { + "precision": 0.6879400814772192, + "recall": 0.5391735201341322, + "f1-score": 0.5201796927640007, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7399563174017385, + "recall": 0.7769089362636157, + "f1-score": 0.709619424742205, + "support": 109341.0 + }, + "cross-entropy": 0.4992874898761024, + "roc-auc": 0.680286897888579, + "score": 0.680286897888579 + }, + "val": { + "0": { + "precision": 0.6876976212482366, + "recall": 0.9110065730119861, + "f1-score": 0.7837560637560638, + "support": 15518.0 + }, + "1": { + "precision": 0.6271598272138229, + "recall": 0.2656982729040375, + "f1-score": 0.3732626335663212, + "support": 8743.0 + }, + "accuracy": 0.6784551337537612, + "macro avg": { + "precision": 0.6574287242310297, + "recall": 0.5883524229580118, + "f1-score": 0.5785093486611925, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6658814581369519, + "recall": 0.6784551337537612, + "f1-score": 0.6358254730900188, + "support": 24261.0 + }, + "cross-entropy": 0.6313069278816857, + "roc-auc": 0.6406711951042247, + "score": 0.6406711951042247 + }, + "test": { + "0": { + "precision": 0.6672369189685388, + "recall": 0.9449828625457983, + "f1-score": 0.7821854822930934, + "support": 16922.0 + }, + "1": { + "precision": 0.6259541984732825, + "recall": 0.16343228784223224, + "f1-score": 0.2591914822824821, + "support": 9533.0 + }, + "accuracy": 0.6633528633528634, + "macro avg": { + "precision": 0.6465955587209107, + "recall": 0.5542075751940153, + "f1-score": 0.5206884822877877, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6523607830970105, + "recall": 0.6633528633528634, + "f1-score": 0.5937257657139531, + "support": 26455.0 + }, + "cross-entropy": 0.6637267283308612, + "roc-auc": 0.6023965414622969, + "score": 0.6023965414622969 + } + }, + "time": "0:01:33.976122", + "diversity": { + "train": { + "mean_centered_corr": -0.013189224526286125, + "mean_pairwise_disagreement": 0.02590141569537856, + "member_std": 0.027156289666891098 + }, + "val": { + "mean_centered_corr": -0.003924679942429066, + "mean_pairwise_disagreement": 0.08662631915552772, + "member_std": 0.060857683420181274 + }, + "test": { + "mean_centered_corr": 0.004035224672406912, + "mean_pairwise_disagreement": 0.054524420653452914, + "member_std": 0.059433531016111374 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:04.881229" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0.toml new file mode 100644 index 000000000..6c0027518 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0/report.json new file mode 100644 index 000000000..5458b2eef --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.786244164176397, + "recall": 0.9782962496584658, + "f1-score": 0.8718187592631802, + "support": 84179.0 + }, + "1": { + "precision": 0.6028260869565217, + "recall": 0.1102058659883952, + "f1-score": 0.18634500369598817, + "support": 25162.0 + }, + "accuracy": 0.7785277251900019, + "macro avg": { + "precision": 0.6945351255664594, + "recall": 0.5442510578234305, + "f1-score": 0.5290818814795841, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7440352429208157, + "recall": 0.7785277251900019, + "f1-score": 0.7140747232878215, + "support": 109341.0 + }, + "cross-entropy": 0.49232123937555056, + "roc-auc": 0.6989090994233629, + "score": 0.6989090994233629 + }, + "val": { + "0": { + "precision": 0.6752969013059107, + "recall": 0.956373243974739, + "f1-score": 0.7916255500733431, + "support": 15518.0 + }, + "1": { + "precision": 0.7035901926444834, + "recall": 0.18380418620610775, + "f1-score": 0.29146640065294277, + "support": 8743.0 + }, + "accuracy": 0.6779605127570999, + "macro avg": { + "precision": 0.689443546975197, + "recall": 0.5700887150904234, + "f1-score": 0.541545975363143, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6854930286779539, + "recall": 0.6779605127570999, + "f1-score": 0.611381889738544, + "support": 24261.0 + }, + "cross-entropy": 0.6870363427483223, + "roc-auc": 0.6227207494642631, + "score": 0.6227207494642631 + }, + "test": { + "0": { + "precision": 0.6564519326065411, + "recall": 0.9785486349131308, + "f1-score": 0.7857736019170997, + "support": 16922.0 + }, + "1": { + "precision": 0.7048780487804878, + "recall": 0.09094723591733976, + "f1-score": 0.1611074979095048, + "support": 9533.0 + }, + "accuracy": 0.6587034587034587, + "macro avg": { + "precision": 0.6806649906935145, + "recall": 0.5347479354152352, + "f1-score": 0.4734405499133022, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6739021751121632, + "recall": 0.6587034587034587, + "f1-score": 0.5606765703728017, + "support": 26455.0 + }, + "cross-entropy": 0.7560552620145878, + "roc-auc": 0.5911612394559284, + "score": 0.5911612394559284 + } + }, + "time": "0:01:33.976122", + "diversity": { + "train": { + "mean_centered_corr": -0.013189224526286125, + "mean_pairwise_disagreement": 0.02590141569537856, + "member_std": 0.027156289666891098 + }, + "val": { + "mean_centered_corr": -0.003924679942429066, + "mean_pairwise_disagreement": 0.08662631915552772, + "member_std": 0.060857683420181274 + }, + "test": { + "mean_centered_corr": 0.004035224672406912, + "mean_pairwise_disagreement": 0.054524420653452914, + "member_std": 0.059433531016111374 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8ff9400b7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,230 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7830293972986266, + "recall": 0.9834400503688568, + "f1-score": 0.8718661632517654, + "support": 84179.0 + }, + "1": { + "precision": 0.6145977329278407, + "recall": 0.0883475081472061, + "f1-score": 0.1544876472427812, + "support": 25162.0 + }, + "accuracy": 0.7774576782725602, + "macro avg": { + "precision": 0.6988135651132337, + "recall": 0.5358937792580315, + "f1-score": 0.5131769052472732, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7442692109193387, + "recall": 0.7774576782725602, + "f1-score": 0.706780072765872, + "support": 109341.0 + }, + "cross-entropy": 0.4979356402925865, + "roc-auc": 0.6847042861139584, + "score": 0.6847042861139584 + }, + "val": { + "0": { + "precision": 0.6772467413674823, + "recall": 0.95424668127336, + "f1-score": 0.7922317631008747, + "support": 15518.0 + }, + "1": { + "precision": 0.7036727879799666, + "recall": 0.19283998627473406, + "f1-score": 0.30272017236735793, + "support": 8743.0 + }, + "accuracy": 0.6798565599109683, + "macro avg": { + "precision": 0.6904597646737245, + "recall": 0.573543333774047, + "f1-score": 0.5474759677341163, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6867699648757033, + "recall": 0.6798565599109683, + "f1-score": 0.6158251913279412, + "support": 24261.0 + }, + "cross-entropy": 0.6400161363311784, + "roc-auc": 0.6415752785241469, + "score": 0.6415752785241469 + }, + "test": { + "0": { + "precision": 0.6584811640422563, + "recall": 0.9761257534570382, + "f1-score": 0.7864403551788988, + "support": 16922.0 + }, + "1": { + "precision": 0.7051094890510949, + "recall": 0.10133221441309137, + "f1-score": 0.17719893607264056, + "support": 9533.0 + }, + "accuracy": 0.6608958608958609, + "macro avg": { + "precision": 0.6817953265466756, + "recall": 0.5387289839350649, + "f1-score": 0.4818196456257697, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6752835765279588, + "recall": 0.6608958608958609, + "f1-score": 0.5669015742928675, + "support": 26455.0 + }, + "cross-entropy": 0.6818542221067515, + "roc-auc": 0.6040898334194843, + "score": 0.6040898334194843 + } + }, + "time": "0:01:33.976122", + "diversity": { + "train": { + "mean_centered_corr": -0.013189224526286125, + "mean_pairwise_disagreement": 0.02590141569537856, + "member_std": 0.027156289666891098 + }, + "val": { + "mean_centered_corr": -0.003924679942429066, + "mean_pairwise_disagreement": 0.08662631915552772, + "member_std": 0.060857683420181274 + }, + "test": { + "mean_centered_corr": 0.004035224672406912, + "mean_pairwise_disagreement": 0.054524420653452914, + "member_std": 0.059433531016111374 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 25 + ], + "head_selection_time": "0:00:16.191151" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..65e0a5aa0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7841306662610283, + "recall": 0.9797930600268475, + "f1-score": 0.8711099845270723, + "support": 84179.0 + }, + "1": { + "precision": 0.5908106807794082, + "recall": 0.09760750337810985, + "f1-score": 0.1675364098366247, + "support": 25162.0 + }, + "accuracy": 0.7767808964615286, + "macro avg": { + "precision": 0.6874706735202183, + "recall": 0.5387002817024786, + "f1-score": 0.5193231971818485, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7396430772076218, + "recall": 0.7767808964615286, + "f1-score": 0.7092007438363795, + "support": 109341.0 + }, + "cross-entropy": 0.4994019756555021, + "roc-auc": 0.6800035420034479, + "score": 0.6800035420034479 + }, + "val": { + "0": { + "precision": 0.6883509833585476, + "recall": 0.9089444516045883, + "f1-score": 0.783415257296787, + "support": 15518.0 + }, + "1": { + "precision": 0.6251989389920425, + "recall": 0.2695870982500286, + "f1-score": 0.376728202669224, + "support": 8743.0 + }, + "accuracy": 0.6785375705865381, + "macro avg": { + "precision": 0.6567749611752951, + "recall": 0.5892657749273085, + "f1-score": 0.5800717299830055, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.665592716020171, + "recall": 0.6785375705865381, + "f1-score": 0.6368563801437931, + "support": 24261.0 + }, + "cross-entropy": 0.6311702903663475, + "roc-auc": 0.6407133660825518, + "score": 0.6407133660825518 + }, + "test": { + "0": { + "precision": 0.6672656609135359, + "recall": 0.9435645904739393, + "f1-score": 0.7817189297691611, + "support": 16922.0 + }, + "1": { + "precision": 0.6219319081551861, + "recall": 0.1647959718871289, + "f1-score": 0.26055228460071317, + "support": 9533.0 + }, + "accuracy": 0.6629370629370629, + "macro avg": { + "precision": 0.644598784534361, + "recall": 0.5541802811805341, + "f1-score": 0.5211356071849371, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6509297446389054, + "recall": 0.6629370629370629, + "f1-score": 0.593917696414755, + "support": 26455.0 + }, + "cross-entropy": 0.6633037626379901, + "roc-auc": 0.6023744607727624, + "score": 0.6023744607727624 + } + }, + "time": "0:01:33.760661", + "diversity": { + "train": { + "mean_centered_corr": -0.012859287671744823, + "mean_pairwise_disagreement": 0.02587178436445202, + "member_std": 0.027145707979798317 + }, + "val": { + "mean_centered_corr": -0.003945968113839626, + "mean_pairwise_disagreement": 0.08649867502735706, + "member_std": 0.061113499104976654 + }, + "test": { + "mean_centered_corr": 0.003282116027548909, + "mean_pairwise_disagreement": 0.05447816133300004, + "member_std": 0.05972920358181 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:04.815423" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0.toml new file mode 100644 index 000000000..69b72ad9f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.01 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0/report.json new file mode 100644 index 000000000..817f2bf62 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7860374117197938, + "recall": 0.9784031646847788, + "f1-score": 0.8717340798797623, + "support": 84179.0 + }, + "1": { + "precision": 0.6014032010524007, + "recall": 0.10901359192433034, + "f1-score": 0.18457087104262693, + "support": 25162.0 + }, + "accuracy": 0.7783356654868714, + "macro avg": { + "precision": 0.6937203063860973, + "recall": 0.5437083783045545, + "f1-score": 0.5281524754611946, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.743548628840426, + "recall": 0.7783356654868714, + "f1-score": 0.7136012599790845, + "support": 109341.0 + }, + "cross-entropy": 0.49236964035190794, + "roc-auc": 0.6988119397829878, + "score": 0.6988119397829878 + }, + "val": { + "0": { + "precision": 0.67549849767823, + "recall": 0.9561799200927955, + "f1-score": 0.7916977910575179, + "support": 15518.0 + }, + "1": { + "precision": 0.7037037037037037, + "recall": 0.18471920393457622, + "f1-score": 0.29262547562964303, + "support": 8743.0 + }, + "accuracy": 0.6781666048390421, + "macro avg": { + "precision": 0.6896011006909668, + "recall": 0.5704495620136859, + "f1-score": 0.5421616333435805, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6856628815156942, + "recall": 0.6781666048390421, + "f1-score": 0.6118457959301155, + "support": 24261.0 + }, + "cross-entropy": 0.686500348338853, + "roc-auc": 0.6227573335158101, + "score": 0.6227573335158101 + }, + "test": { + "0": { + "precision": 0.6564879441624365, + "recall": 0.9782531615648269, + "f1-score": 0.7857041150505482, + "support": 16922.0 + }, + "1": { + "precision": 0.7029862792574657, + "recall": 0.0913668310080772, + "f1-score": 0.1617155588562941, + "support": 9533.0 + }, + "accuracy": 0.6586656586656586, + "macro avg": { + "precision": 0.6797371117099511, + "recall": 0.5348099962864521, + "f1-score": 0.4737098369534211, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.67324351507383, + "recall": 0.6586656586656586, + "f1-score": 0.5608512363433161, + "support": 26455.0 + }, + "cross-entropy": 0.7551689618359527, + "roc-auc": 0.5911827002496308, + "score": 0.5911827002496308 + } + }, + "time": "0:01:33.760661", + "diversity": { + "train": { + "mean_centered_corr": -0.012859287671744823, + "mean_pairwise_disagreement": 0.02587178436445202, + "member_std": 0.027145707979798317 + }, + "val": { + "mean_centered_corr": -0.003945968113839626, + "mean_pairwise_disagreement": 0.08649867502735706, + "member_std": 0.061113499104976654 + }, + "test": { + "mean_centered_corr": 0.003282116027548909, + "mean_pairwise_disagreement": 0.05447816133300004, + "member_std": 0.05972920358181 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..357dac6a7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,230 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7828372603154192, + "recall": 0.983570724289906, + "f1-score": 0.8717983816197491, + "support": 84179.0 + }, + "1": { + "precision": 0.6133631534805704, + "recall": 0.0871949765519434, + "f1-score": 0.15268450537596995, + "support": 25162.0 + }, + "accuracy": 0.7772930556698768, + "macro avg": { + "precision": 0.6981002068979948, + "recall": 0.5353828504209247, + "f1-score": 0.5122414434978595, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7438371827948325, + "recall": 0.7772930556698768, + "f1-score": 0.7063129429092382, + "support": 109341.0 + }, + "cross-entropy": 0.49804174438759463, + "roc-auc": 0.6845043493304455, + "score": 0.6845043493304455 + }, + "val": { + "0": { + "precision": 0.6775611095852788, + "recall": 0.9538600335094729, + "f1-score": 0.7923134568033401, + "support": 15518.0 + }, + "1": { + "precision": 0.7035196687370601, + "recall": 0.19432689008349538, + "f1-score": 0.3045348628786521, + "support": 8743.0 + }, + "accuracy": 0.6801450888256874, + "macro avg": { + "precision": 0.6905403891611694, + "recall": 0.5740934617964841, + "f1-score": 0.5484241598409961, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6869158634150477, + "recall": 0.6801450888256874, + "f1-score": 0.616531409621297, + "support": 24261.0 + }, + "cross-entropy": 0.6396176757776643, + "roc-auc": 0.6415860912175324, + "score": 0.6415860912175324 + }, + "test": { + "0": { + "precision": 0.6586747516655364, + "recall": 0.9757120907694126, + "f1-score": 0.7864440686846556, + "support": 16922.0 + }, + "1": { + "precision": 0.7038904899135446, + "recall": 0.10248610091261932, + "f1-score": 0.17892134419924915, + "support": 9533.0 + }, + "accuracy": 0.661047061047061, + "macro avg": { + "precision": 0.6812826207895405, + "recall": 0.5390990958410159, + "f1-score": 0.4826827064419523, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6749681416756766, + "recall": 0.661047061047061, + "f1-score": 0.5675246155559699, + "support": 26455.0 + }, + "cross-entropy": 0.6812321989565008, + "roc-auc": 0.6040412707800087, + "score": 0.6040412707800087 + } + }, + "time": "0:01:33.760661", + "diversity": { + "train": { + "mean_centered_corr": -0.012859287671744823, + "mean_pairwise_disagreement": 0.02587178436445202, + "member_std": 0.027145707979798317 + }, + "val": { + "mean_centered_corr": -0.003945968113839626, + "mean_pairwise_disagreement": 0.08649867502735706, + "member_std": 0.061113499104976654 + }, + "test": { + "mean_centered_corr": 0.003282116027548909, + "mean_pairwise_disagreement": 0.05447816133300004, + "member_std": 0.05972920358181 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 25 + ], + "head_selection_time": "0:00:15.930728" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8fc29a25e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,426 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7889205040357703, + "recall": 0.9683650316587272, + "f1-score": 0.8694808138449641, + "support": 84179.0 + }, + "1": { + "precision": 0.5572734829592685, + "recall": 0.13321675542484698, + "f1-score": 0.21503031080604293, + "support": 25162.0 + }, + "accuracy": 0.7761772802516896, + "macro avg": { + "precision": 0.6730969934975194, + "recall": 0.5507908935417871, + "f1-score": 0.5422555623255035, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7356129401363461, + "recall": 0.7761772802516896, + "f1-score": 0.7188759761585946, + "support": 109341.0 + }, + "cross-entropy": 0.4988956148214662, + "roc-auc": 0.68250001764071, + "score": 0.68250001764071 + }, + "val": { + "0": { + "precision": 0.6950127877237852, + "recall": 0.910619925248099, + "f1-score": 0.7883403068340307, + "support": 15518.0 + }, + "1": { + "precision": 0.6469839653855943, + "recall": 0.2907468832208624, + "f1-score": 0.401199494949495, + "support": 8743.0 + }, + "accuracy": 0.6872346564444994, + "macro avg": { + "precision": 0.6709983765546897, + "recall": 0.6006834042344806, + "f1-score": 0.5947699008917628, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6777045154471766, + "recall": 0.6872346564444994, + "f1-score": 0.6488253602816835, + "support": 24261.0 + }, + "cross-entropy": 0.6245548911838499, + "roc-auc": 0.6477890245840551, + "score": 0.6477890245840551 + }, + "test": { + "0": { + "precision": 0.6679904029122198, + "recall": 0.9542607256825434, + "f1-score": 0.7858672376873662, + "support": 16922.0 + }, + "1": { + "precision": 0.6606751424813678, + "recall": 0.1580824504353299, + "f1-score": 0.25512104283054005, + "support": 9533.0 + }, + "accuracy": 0.6673596673596673, + "macro avg": { + "precision": 0.6643327726967938, + "recall": 0.5561715880589366, + "f1-score": 0.5204941402589531, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6653543651996018, + "recall": 0.6673596673596673, + "f1-score": 0.5946140350576129, + "support": 26455.0 + }, + "cross-entropy": 0.6584188876218103, + "roc-auc": 0.6045088303107439, + "score": 0.6045088303107439 + } + }, + "time": "0:01:18.654365", + "diversity": { + "train": { + "mean_centered_corr": -0.01063462533056736, + "mean_pairwise_disagreement": 0.026665487313881315, + "member_std": 0.029800426214933395 + }, + "val": { + "mean_centered_corr": -0.013281173072755337, + "mean_pairwise_disagreement": 0.09412748922670262, + "member_std": 0.06749773025512695 + }, + "test": { + "mean_centered_corr": -0.011601177044212818, + "mean_pairwise_disagreement": 0.059119563958273634, + "member_std": 0.06550896167755127 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:04.756683" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..cd8d74155 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0.toml @@ -0,0 +1,57 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..5e1e2aae3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0/report.json @@ -0,0 +1,422 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7842480378851866, + "recall": 0.9816818921583768, + "f1-score": 0.8719282511210762, + "support": 84179.0 + }, + "1": { + "precision": 0.6115869017632242, + "recall": 0.09649471425164931, + "f1-score": 0.1666895510091995, + "support": 25162.0 + }, + "accuracy": 0.7779789831810574, + "macro avg": { + "precision": 0.6979174698242054, + "recall": 0.539088303205013, + "f1-score": 0.5193089010651378, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7445145480954387, + "recall": 0.7779789831810574, + "f1-score": 0.7096358249294825, + "support": 109341.0 + }, + "cross-entropy": 0.4927771605769271, + "roc-auc": 0.6981357694476362, + "score": 0.6981357694476362 + }, + "val": { + "0": { + "precision": 0.6767607563716087, + "recall": 0.9548266529191907, + "f1-score": 0.7920987918314979, + "support": 15518.0 + }, + "1": { + "precision": 0.7038445289395859, + "recall": 0.19055244195356286, + "f1-score": 0.29990999099909993, + "support": 8743.0 + }, + "accuracy": 0.6794031573306953, + "macro avg": { + "precision": 0.6903026426555974, + "recall": 0.5726895474363768, + "f1-score": 0.5460043914152989, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6865210063020248, + "recall": 0.6794031573306953, + "f1-score": 0.6147274268557072, + "support": 24261.0 + }, + "cross-entropy": 0.673915591317614, + "roc-auc": 0.6250550271749447, + "score": 0.6250550271749447 + }, + "test": { + "0": { + "precision": 0.657534791252485, + "recall": 0.9772485521805933, + "f1-score": 0.7861285415478227, + "support": 16922.0 + }, + "1": { + "precision": 0.7049808429118773, + "recall": 0.09650687086961082, + "f1-score": 0.16977302085255583, + "support": 9533.0 + }, + "accuracy": 0.6598752598752599, + "macro avg": { + "precision": 0.6812578170821812, + "recall": 0.536877711525102, + "f1-score": 0.4779507812001893, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6746318697052912, + "recall": 0.6598752598752599, + "f1-score": 0.5640262100873057, + "support": 26455.0 + }, + "cross-entropy": 0.735759481729751, + "roc-auc": 0.592670158895295, + "score": 0.592670158895295 + } + }, + "time": "0:01:18.654365", + "diversity": { + "train": { + "mean_centered_corr": -0.01063462533056736, + "mean_pairwise_disagreement": 0.026665487313881315, + "member_std": 0.029800426214933395 + }, + "val": { + "mean_centered_corr": -0.013281173072755337, + "mean_pairwise_disagreement": 0.09412748922670262, + "member_std": 0.06749773025512695 + }, + "test": { + "mean_centered_corr": -0.011601177044212818, + "mean_pairwise_disagreement": 0.059119563958273634, + "member_std": 0.06550896167755127 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5f7214ce5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,428 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7909659567005086, + "recall": 0.9643735373430428, + "f1-score": 0.8691043985161632, + "support": 84179.0 + }, + "1": { + "precision": 0.5528552258834054, + "recall": 0.14736507431841667, + "f1-score": 0.23270262637672973, + "support": 25162.0 + }, + "accuracy": 0.7763601942546712, + "macro avg": { + "precision": 0.671910591291957, + "recall": 0.5558693058307297, + "f1-score": 0.5509035124464464, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7361709373681451, + "recall": 0.7763601942546712, + "f1-score": 0.7226530089132472, + "support": 109341.0 + }, + "cross-entropy": 0.5000394688239905, + "roc-auc": 0.6809790060024956, + "score": 0.6809790060024956 + }, + "val": { + "0": { + "precision": 0.7075808427894092, + "recall": 0.8559092666580745, + "f1-score": 0.7747090903788387, + "support": 15518.0 + }, + "1": { + "precision": 0.5927140255009108, + "recall": 0.3721834610545579, + "f1-score": 0.4572472423241762, + "support": 8743.0 + }, + "accuracy": 0.6815877333992828, + "macro avg": { + "precision": 0.6501474341451601, + "recall": 0.6140463638563163, + "f1-score": 0.6159781663515075, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6661859875256797, + "recall": 0.6815877333992828, + "f1-score": 0.6603045341964096, + "support": 24261.0 + }, + "cross-entropy": 0.6203779736299148, + "roc-auc": 0.6506783465179156, + "score": 0.6506783465179156 + }, + "test": { + "0": { + "precision": 0.6800559465011583, + "recall": 0.9194539652523342, + "f1-score": 0.7818396522700435, + "support": 16922.0 + }, + "1": { + "precision": 0.618847874720358, + "recall": 0.23214098395048777, + "f1-score": 0.3376306354412999, + "support": 9533.0 + }, + "accuracy": 0.6717822717822718, + "macro avg": { + "precision": 0.6494519106107581, + "recall": 0.575797474601411, + "f1-score": 0.5597351438556717, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6579997549197419, + "recall": 0.6717822717822718, + "f1-score": 0.6217699279295251, + "support": 26455.0 + }, + "cross-entropy": 0.6443009499315193, + "roc-auc": 0.6065618416202598, + "score": 0.6065618416202598 + } + }, + "time": "0:01:18.654365", + "diversity": { + "train": { + "mean_centered_corr": -0.01063462533056736, + "mean_pairwise_disagreement": 0.026665487313881315, + "member_std": 0.029800426214933395 + }, + "val": { + "mean_centered_corr": -0.013281173072755337, + "mean_pairwise_disagreement": 0.09412748922670262, + "member_std": 0.06749773025512695 + }, + "test": { + "mean_centered_corr": -0.011601177044212818, + "mean_pairwise_disagreement": 0.059119563958273634, + "member_std": 0.06550896167755127 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 24, + 24 + ], + "head_selection_time": "0:00:20.435848" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-best-head-evaluation/0/report.json new file mode 100644 index 000000000..85c7d6ec0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-best-head-evaluation/0/report.json @@ -0,0 +1,255 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.6894886493682861, + "min_features_per_member": 67.0, + "mean_features_per_member": 75.84375, + "max_features_per_member": 84.0, + "min_members_per_feature": 14.0, + "mean_members_per_feature": 22.063636779785156, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.5278457608823273, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7894966160281175, + "recall": 0.9686501383955618, + "f1-score": 0.8699455883921904, + "support": 84179.0 + }, + "1": { + "precision": 0.5645214521452145, + "recall": 0.13595898577219617, + "f1-score": 0.21914034975337904, + "support": 25162.0 + }, + "accuracy": 0.7770278303655537, + "macro avg": { + "precision": 0.677009034086666, + "recall": 0.552304562083879, + "f1-score": 0.5445429690727848, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7377244073084094, + "recall": 0.7770278303655537, + "f1-score": 0.7201796139212255, + "support": 109341.0 + }, + "cross-entropy": 0.5002029863194409, + "roc-auc": 0.6821879786641952, + "score": 0.6821879786641952 + }, + "val": { + "0": { + "precision": 0.6961402811584124, + "recall": 0.9030802938523006, + "f1-score": 0.7862212123762238, + "support": 15518.0 + }, + "1": { + "precision": 0.6358353510895883, + "recall": 0.30035456936978155, + "f1-score": 0.4079857065175173, + "support": 8743.0 + }, + "accuracy": 0.6858744487036808, + "macro avg": { + "precision": 0.6659878161240004, + "recall": 0.6017174316110411, + "f1-score": 0.5971034594468705, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6744080358432264, + "recall": 0.6858744487036808, + "f1-score": 0.6499154942391862, + "support": 24261.0 + }, + "cross-entropy": 0.6154470541736884, + "roc-auc": 0.6547032702847417, + "score": 0.6547032702847417 + }, + "test": { + "0": { + "precision": 0.6733296707940667, + "recall": 0.9415553717054722, + "f1-score": 0.7851669335961562, + "support": 16922.0 + }, + "1": { + "precision": 0.6457736389684814, + "recall": 0.18913248714990036, + "f1-score": 0.2925760649087221, + "support": 9533.0 + }, + "accuracy": 0.6704214704214704, + "macro avg": { + "precision": 0.6595516548812741, + "recall": 0.5653439294276863, + "f1-score": 0.5388714992524392, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6633999164416454, + "recall": 0.6704214704214704, + "f1-score": 0.6076629172968816, + "support": 26455.0 + }, + "cross-entropy": 0.638677137478482, + "roc-auc": 0.6118564029158264, + "score": 0.6118564029158264 + } + }, + "time": "0:01:10.628940", + "diversity": { + "train": { + "mean_centered_corr": -0.015634549781680107, + "mean_pairwise_disagreement": 0.03130809179096706, + "member_std": 0.0355970673263073 + }, + "val": { + "mean_centered_corr": -0.0187542624771595, + "mean_pairwise_disagreement": 0.12205770312369114, + "member_std": 0.07598911970853806 + }, + "test": { + "mean_centered_corr": -0.018549803644418716, + "mean_pairwise_disagreement": 0.08149855811146134, + "member_std": 0.0770004391670227 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:04.834038" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0.toml new file mode 100644 index 000000000..5319a963b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.7 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0/report.json new file mode 100644 index 000000000..d3d96f1bb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0/report.json @@ -0,0 +1,251 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.6894886493682861, + "min_features_per_member": 67.0, + "mean_features_per_member": 75.84375, + "max_features_per_member": 84.0, + "min_members_per_feature": 14.0, + "mean_members_per_feature": 22.063636779785156, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.5278457608823273, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7830881310769057, + "recall": 0.9845092006319865, + "f1-score": 0.8723225093416136, + "support": 84179.0 + }, + "1": { + "precision": 0.6284900284900284, + "recall": 0.08767188617756935, + "f1-score": 0.15387834821428573, + "support": 25162.0 + }, + "accuracy": 0.7781253143834427, + "macro avg": { + "precision": 0.705789079783467, + "recall": 0.5360905434047779, + "f1-score": 0.5131004287779497, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7475113807518583, + "recall": 0.7781253143834427, + "f1-score": 0.7069911882243216, + "support": 109341.0 + }, + "cross-entropy": 0.4934543091460455, + "roc-auc": 0.6961052398986505, + "score": 0.6961052398986505 + }, + "val": { + "0": { + "precision": 0.6825136099117702, + "recall": 0.9371697383683464, + "f1-score": 0.7898224080812469, + "support": 15518.0 + }, + "1": { + "precision": 0.6698272942770064, + "recall": 0.22623813336383392, + "f1-score": 0.3382352941176471, + "support": 8743.0 + }, + "accuracy": 0.6809694571534561, + "macro avg": { + "precision": 0.6761704520943883, + "recall": 0.5817039358660901, + "f1-score": 0.564028851099447, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6779418091782992, + "recall": 0.6809694571534561, + "f1-score": 0.627082779154832, + "support": 24261.0 + }, + "cross-entropy": 0.6467973900252932, + "roc-auc": 0.6291199291618959, + "score": 0.6291199291618959 + }, + "test": { + "0": { + "precision": 0.661515493867011, + "recall": 0.9688571090887602, + "f1-score": 0.7862178103869947, + "support": 16922.0 + }, + "1": { + "precision": 0.6846199880311191, + "recall": 0.12000419595090737, + "f1-score": 0.20421278114958943, + "support": 9533.0 + }, + "accuracy": 0.662974862974863, + "macro avg": { + "precision": 0.6730677409490651, + "recall": 0.5444306525198338, + "f1-score": 0.49521529576829204, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6698411465930153, + "recall": 0.662974862974863, + "f1-score": 0.5764936015901629, + "support": 26455.0 + }, + "cross-entropy": 0.6863610680565896, + "roc-auc": 0.5961977102213372, + "score": 0.5961977102213372 + } + }, + "time": "0:01:10.628940", + "diversity": { + "train": { + "mean_centered_corr": -0.015634549781680107, + "mean_pairwise_disagreement": 0.03130809179096706, + "member_std": 0.0355970673263073 + }, + "val": { + "mean_centered_corr": -0.0187542624771595, + "mean_pairwise_disagreement": 0.12205770312369114, + "member_std": 0.07598911970853806 + }, + "test": { + "mean_centered_corr": -0.018549803644418716, + "mean_pairwise_disagreement": 0.08149855811146134, + "member_std": 0.0770004391670227 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d21da633b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.7-greedy-heads-evaluation/0/report.json @@ -0,0 +1,255 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.6894886493682861, + "min_features_per_member": 67.0, + "mean_features_per_member": 75.84375, + "max_features_per_member": 84.0, + "min_members_per_feature": 14.0, + "mean_members_per_feature": 22.063636779785156, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.5278457608823273, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 428, + "metrics": { + "train": { + "0": { + "precision": 0.7894966160281175, + "recall": 0.9686501383955618, + "f1-score": 0.8699455883921904, + "support": 84179.0 + }, + "1": { + "precision": 0.5645214521452145, + "recall": 0.13595898577219617, + "f1-score": 0.21914034975337904, + "support": 25162.0 + }, + "accuracy": 0.7770278303655537, + "macro avg": { + "precision": 0.677009034086666, + "recall": 0.552304562083879, + "f1-score": 0.5445429690727848, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7377244073084094, + "recall": 0.7770278303655537, + "f1-score": 0.7201796139212255, + "support": 109341.0 + }, + "cross-entropy": 0.5002029863194409, + "roc-auc": 0.6821879786641952, + "score": 0.6821879786641952 + }, + "val": { + "0": { + "precision": 0.6961402811584124, + "recall": 0.9030802938523006, + "f1-score": 0.7862212123762238, + "support": 15518.0 + }, + "1": { + "precision": 0.6358353510895883, + "recall": 0.30035456936978155, + "f1-score": 0.4079857065175173, + "support": 8743.0 + }, + "accuracy": 0.6858744487036808, + "macro avg": { + "precision": 0.6659878161240004, + "recall": 0.6017174316110411, + "f1-score": 0.5971034594468705, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6744080358432264, + "recall": 0.6858744487036808, + "f1-score": 0.6499154942391862, + "support": 24261.0 + }, + "cross-entropy": 0.6154470541736884, + "roc-auc": 0.6547032702847417, + "score": 0.6547032702847417 + }, + "test": { + "0": { + "precision": 0.6733296707940667, + "recall": 0.9415553717054722, + "f1-score": 0.7851669335961562, + "support": 16922.0 + }, + "1": { + "precision": 0.6457736389684814, + "recall": 0.18913248714990036, + "f1-score": 0.2925760649087221, + "support": 9533.0 + }, + "accuracy": 0.6704214704214704, + "macro avg": { + "precision": 0.6595516548812741, + "recall": 0.5653439294276863, + "f1-score": 0.5388714992524392, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6633999164416454, + "recall": 0.6704214704214704, + "f1-score": 0.6076629172968816, + "support": 26455.0 + }, + "cross-entropy": 0.638677137478482, + "roc-auc": 0.6118564029158264, + "score": 0.6118564029158264 + } + }, + "time": "0:01:10.628940", + "diversity": { + "train": { + "mean_centered_corr": -0.015634549781680107, + "mean_pairwise_disagreement": 0.03130809179096706, + "member_std": 0.0355970673263073 + }, + "val": { + "mean_centered_corr": -0.0187542624771595, + "mean_pairwise_disagreement": 0.12205770312369114, + "member_std": 0.07598911970853806 + }, + "test": { + "mean_centered_corr": -0.018549803644418716, + "mean_pairwise_disagreement": 0.08149855811146134, + "member_std": 0.0770004391670227 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:11.162732" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0237bf07a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-best-head-evaluation/0/report.json @@ -0,0 +1,255 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7894886136054993, + "min_features_per_member": 82.0, + "mean_features_per_member": 86.84375, + "max_features_per_member": 93.0, + "min_members_per_feature": 18.0, + "mean_members_per_feature": 25.263635635375977, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6542975594370393, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7923570623683308, + "recall": 0.9606196319747206, + "f1-score": 0.8684128569434153, + "support": 84179.0 + }, + "1": { + "precision": 0.5450178424375515, + "recall": 0.15781734361338526, + "f1-score": 0.24476084812623275, + "support": 25162.0 + }, + "accuracy": 0.7758754721467702, + "macro avg": { + "precision": 0.6686874524029411, + "recall": 0.5592184877940529, + "f1-score": 0.556586852534824, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7354383452183298, + "recall": 0.7758754721467702, + "f1-score": 0.7248954952414192, + "support": 109341.0 + }, + "cross-entropy": 0.5019736660581554, + "roc-auc": 0.679122291624921, + "score": 0.679122291624921 + }, + "val": { + "0": { + "precision": 0.7086274721129493, + "recall": 0.8474030158525583, + "f1-score": 0.7718268525311812, + "support": 15518.0 + }, + "1": { + "precision": 0.5848527349228612, + "recall": 0.38156239277135995, + "f1-score": 0.4618259846334879, + "support": 8743.0 + }, + "accuracy": 0.6795268125798607, + "macro avg": { + "precision": 0.6467401035179052, + "recall": 0.6144827043119592, + "f1-score": 0.6168264185823346, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6640224464646685, + "recall": 0.6795268125798607, + "f1-score": 0.6601110292745335, + "support": 24261.0 + }, + "cross-entropy": 0.6209465653764391, + "roc-auc": 0.652377048657135, + "score": 0.652377048657135 + }, + "test": { + "0": { + "precision": 0.6830873199359773, + "recall": 0.9079305046684789, + "f1-score": 0.7796214543055767, + "support": 16922.0 + }, + "1": { + "precision": 0.6068634872571285, + "recall": 0.25228154830588484, + "f1-score": 0.3564018968583284, + "support": 9533.0 + }, + "accuracy": 0.6716688716688717, + "macro avg": { + "precision": 0.6449754035965529, + "recall": 0.5801060264871819, + "f1-score": 0.5680116755819525, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6556202325450318, + "recall": 0.6716688716688717, + "f1-score": 0.6271152346440906, + "support": 26455.0 + }, + "cross-entropy": 0.6427730061058482, + "roc-auc": 0.6066780069996902, + "score": 0.6066780069996902 + } + }, + "time": "0:01:07.254601", + "diversity": { + "train": { + "mean_centered_corr": -0.014071253128349781, + "mean_pairwise_disagreement": 0.02823395645053607, + "member_std": 0.03310052305459976 + }, + "val": { + "mean_centered_corr": -0.021914731711149216, + "mean_pairwise_disagreement": 0.10051144076980047, + "member_std": 0.07142822444438934 + }, + "test": { + "mean_centered_corr": -0.023642798885703087, + "mean_pairwise_disagreement": 0.0652579547740838, + "member_std": 0.07157552242279053 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24 + ], + "head_selection_time": "0:00:04.881467" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0.toml new file mode 100644 index 000000000..5c75f01be --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.8 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0/report.json new file mode 100644 index 000000000..92fd4b88b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-evaluation/0/report.json @@ -0,0 +1,251 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7894886136054993, + "min_features_per_member": 82.0, + "mean_features_per_member": 86.84375, + "max_features_per_member": 93.0, + "min_members_per_feature": 18.0, + "mean_members_per_feature": 25.263635635375977, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6542975594370393, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7822859783644097, + "recall": 0.9853407619477542, + "f1-score": 0.8721504878869448, + "support": 84179.0 + }, + "1": { + "precision": 0.6274154589371981, + "recall": 0.08258485017089262, + "f1-score": 0.1459577158109152, + "support": 25162.0 + }, + "accuracy": 0.7775948637747963, + "macro avg": { + "precision": 0.704850718650804, + "recall": 0.5339628060593234, + "f1-score": 0.5090541018489301, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7466465383572076, + "recall": 0.7775948637747963, + "f1-score": 0.7050360245934222, + "support": 109341.0 + }, + "cross-entropy": 0.49400328229016993, + "roc-auc": 0.6954429108993698, + "score": 0.6954429108993698 + }, + "val": { + "0": { + "precision": 0.6779777757369823, + "recall": 0.9514757056321691, + "f1-score": 0.7917739167739167, + "support": 15518.0 + }, + "1": { + "precision": 0.6967378171566653, + "recall": 0.19787258378131076, + "f1-score": 0.30821307678603244, + "support": 8743.0 + }, + "accuracy": 0.6798977783273566, + "macro avg": { + "precision": 0.6873577964468238, + "recall": 0.5746741447067399, + "f1-score": 0.5499934967799746, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6847383809112244, + "recall": 0.6798977783273566, + "f1-score": 0.6175118326053305, + "support": 24261.0 + }, + "cross-entropy": 0.6534319461189932, + "roc-auc": 0.6293880905914133, + "score": 0.6293880905914133 + }, + "test": { + "0": { + "precision": 0.6589364254298281, + "recall": 0.9738801560099279, + "f1-score": 0.7860345320995898, + "support": 16922.0 + }, + "1": { + "precision": 0.6941176470588235, + "recall": 0.10521346900241267, + "f1-score": 0.18272909455274186, + "support": 9533.0 + }, + "accuracy": 0.6608580608580609, + "macro avg": { + "precision": 0.6765270362443259, + "recall": 0.5395468125061703, + "f1-score": 0.48438181332616587, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6716138998501348, + "recall": 0.6608580608580609, + "f1-score": 0.5686347688739576, + "support": 26455.0 + }, + "cross-entropy": 0.7009929789782234, + "roc-auc": 0.5962685705138886, + "score": 0.5962685705138886 + } + }, + "time": "0:01:07.254601", + "diversity": { + "train": { + "mean_centered_corr": -0.014071253128349781, + "mean_pairwise_disagreement": 0.02823395645053607, + "member_std": 0.03310052305459976 + }, + "val": { + "mean_centered_corr": -0.021914731711149216, + "mean_pairwise_disagreement": 0.10051144076980047, + "member_std": 0.07142822444438934 + }, + "test": { + "mean_centered_corr": -0.023642798885703087, + "mean_pairwise_disagreement": 0.0652579547740838, + "member_std": 0.07157552242279053 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..e66cb847a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.8-greedy-heads-evaluation/0/report.json @@ -0,0 +1,256 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7894886136054993, + "min_features_per_member": 82.0, + "mean_features_per_member": 86.84375, + "max_features_per_member": 93.0, + "min_members_per_feature": 18.0, + "mean_members_per_feature": 25.263635635375977, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6542975594370393, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7922584047977184, + "recall": 0.9635776143693795, + "f1-score": 0.8695600902653824, + "support": 84179.0 + }, + "1": { + "precision": 0.5594194568185084, + "recall": 0.15471743104681662, + "f1-score": 0.24239594035054948, + "support": 25162.0 + }, + "accuracy": 0.777439386872262, + "macro avg": { + "precision": 0.6758389308081134, + "recall": 0.5591475227080981, + "f1-score": 0.555978015307966, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7386765497840191, + "recall": 0.777439386872262, + "f1-score": 0.7252345002291012, + "support": 109341.0 + }, + "cross-entropy": 0.4993136444932815, + "roc-auc": 0.6856466838728517, + "score": 0.6856466838728517 + }, + "val": { + "0": { + "precision": 0.7056227009984235, + "recall": 0.8653176955793273, + "f1-score": 0.7773532476554359, + "support": 15518.0 + }, + "1": { + "precision": 0.6004588032880902, + "recall": 0.35925883563994054, + "f1-score": 0.44954916273078577, + "support": 8743.0 + }, + "accuracy": 0.6829479411401014, + "macro avg": { + "precision": 0.6530407521432569, + "recall": 0.6122882656096339, + "f1-score": 0.6134512051931108, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6677245122312069, + "recall": 0.6829479411401014, + "f1-score": 0.659221632532555, + "support": 24261.0 + }, + "cross-entropy": 0.6165028482842303, + "roc-auc": 0.6540790454616192, + "score": 0.6540790454616192 + }, + "test": { + "0": { + "precision": 0.6807331442286711, + "recall": 0.9218177520387661, + "f1-score": 0.7831413008007632, + "support": 16922.0 + }, + "1": { + "precision": 0.6262711864406779, + "recall": 0.2325605790412252, + "f1-score": 0.3391723399372753, + "support": 9533.0 + }, + "accuracy": 0.6734454734454735, + "macro avg": { + "precision": 0.6535021653346745, + "recall": 0.5771891655399957, + "f1-score": 0.5611568203690193, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6611078997156135, + "recall": 0.6734454734454735, + "f1-score": 0.6231580800896828, + "support": 26455.0 + }, + "cross-entropy": 0.635872441430285, + "roc-auc": 0.6102977368359448, + "score": 0.6102977368359448 + } + }, + "time": "0:01:07.254601", + "diversity": { + "train": { + "mean_centered_corr": -0.014071253128349781, + "mean_pairwise_disagreement": 0.02823395645053607, + "member_std": 0.03310052305459976 + }, + "val": { + "mean_centered_corr": -0.021914731711149216, + "mean_pairwise_disagreement": 0.10051144076980047, + "member_std": 0.07142822444438934 + }, + "test": { + "mean_centered_corr": -0.023642798885703087, + "mean_pairwise_disagreement": 0.0652579547740838, + "member_std": 0.07157552242279053 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24, + 11 + ], + "head_selection_time": "0:00:15.939899" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-best-head-evaluation/0/report.json new file mode 100644 index 000000000..841a9dab2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-best-head-evaluation/0/report.json @@ -0,0 +1,255 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7919161090894863, + "recall": 0.9617125411325865, + "f1-score": 0.8685939905690236, + "support": 84179.0 + }, + "1": { + "precision": 0.5468859834106565, + "recall": 0.15459820364041013, + "f1-score": 0.24105344694035633, + "support": 25162.0 + }, + "accuracy": 0.77597607484841, + "macro avg": { + "precision": 0.6694010462500715, + "recall": 0.5581553723864984, + "f1-score": 0.5548237187546899, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7355287701925426, + "recall": 0.77597607484841, + "f1-score": 0.7241817832654089, + "support": 109341.0 + }, + "cross-entropy": 0.5019382114215071, + "roc-auc": 0.6778537614421274, + "score": 0.6778537614421274 + }, + "val": { + "0": { + "precision": 0.7093391589295467, + "recall": 0.8369635262276066, + "f1-score": 0.7678845926451461, + "support": 15518.0 + }, + "1": { + "precision": 0.5748613678373382, + "recall": 0.39128445613633767, + "f1-score": 0.46563223084252076, + "support": 8743.0 + }, + "accuracy": 0.6763529945179506, + "macro avg": { + "precision": 0.6421002633834425, + "recall": 0.6141239911819721, + "f1-score": 0.6167584117438334, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.660877045763594, + "recall": 0.6763529945179506, + "f1-score": 0.6589611187883243, + "support": 24261.0 + }, + "cross-entropy": 0.6229299585114317, + "roc-auc": 0.6501781765294031, + "score": 0.6501781765294031 + }, + "test": { + "0": { + "precision": 0.6828953229398663, + "recall": 0.9059803805696726, + "f1-score": 0.7787767956923702, + "support": 16922.0 + }, + "1": { + "precision": 0.6027465667915106, + "recall": 0.25322563726004405, + "f1-score": 0.3566257940611612, + "support": 9533.0 + }, + "accuracy": 0.6707616707616708, + "macro avg": { + "precision": 0.6428209448656885, + "recall": 0.5796030089148583, + "f1-score": 0.5677012948767657, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6540138981671476, + "recall": 0.6707616707616708, + "f1-score": 0.6266556277260004, + "support": 26455.0 + }, + "cross-entropy": 0.6442264400252198, + "roc-auc": 0.6048261425892079, + "score": 0.6048261425892079 + } + }, + "time": "0:01:07.634554", + "diversity": { + "train": { + "mean_centered_corr": -0.010898238979279995, + "mean_pairwise_disagreement": 0.026737251557793006, + "member_std": 0.030055008828639984 + }, + "val": { + "mean_centered_corr": -0.008359640836715698, + "mean_pairwise_disagreement": 0.09315727751030128, + "member_std": 0.06754488497972488 + }, + "test": { + "mean_centered_corr": -0.005901505704969168, + "mean_pairwise_disagreement": 0.05854524725492467, + "member_std": 0.06570815294981003 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24 + ], + "head_selection_time": "0:00:04.826316" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0.toml new file mode 100644 index 000000000..54bfca637 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0/report.json new file mode 100644 index 000000000..301ce004d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-evaluation/0/report.json @@ -0,0 +1,251 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7843021876334646, + "recall": 0.9816937716057449, + "f1-score": 0.8719664035791161, + "support": 84179.0 + }, + "1": { + "precision": 0.6124245472837022, + "recall": 0.09677291153326445, + "f1-score": 0.16713569908710274, + "support": 25162.0 + }, + "accuracy": 0.77805214878225, + "macro avg": { + "precision": 0.6983633674585834, + "recall": 0.5392333415695046, + "f1-score": 0.5195510513331094, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7447489991087509, + "recall": 0.77805214878225, + "f1-score": 0.7097678670152651, + "support": 109341.0 + }, + "cross-entropy": 0.4927182387508514, + "roc-auc": 0.698241002315497, + "score": 0.698241002315497 + }, + "val": { + "0": { + "precision": 0.6764142335766423, + "recall": 0.9554710658590024, + "f1-score": 0.792082910411881, + "support": 15518.0 + }, + "1": { + "precision": 0.7048269970098249, + "recall": 0.18872240649662586, + "f1-score": 0.2977264525442079, + "support": 8743.0 + }, + "accuracy": 0.6791558468323647, + "macro avg": { + "precision": 0.6906206152932336, + "recall": 0.5720967361778142, + "f1-score": 0.5449046814780444, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6866534154197781, + "recall": 0.6791558468323647, + "f1-score": 0.6139303812029833, + "support": 24261.0 + }, + "cross-entropy": 0.6745751820826075, + "roc-auc": 0.6250090898119413, + "score": 0.6250090898119413 + }, + "test": { + "0": { + "precision": 0.6571337782014617, + "recall": 0.9776622148682189, + "f1-score": 0.7859755807876859, + "support": 16922.0 + }, + "1": { + "precision": 0.7044566067240031, + "recall": 0.09451379418860799, + "f1-score": 0.16666666666666666, + "support": 9533.0 + }, + "accuracy": 0.6594216594216594, + "macro avg": { + "precision": 0.6807951924627325, + "recall": 0.5360880045284134, + "f1-score": 0.4763211237271763, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6741864534728805, + "recall": 0.6594216594216594, + "f1-score": 0.5628090006207732, + "support": 26455.0 + }, + "cross-entropy": 0.7367691279234707, + "roc-auc": 0.5927112610884333, + "score": 0.5927112610884333 + } + }, + "time": "0:01:07.634554", + "diversity": { + "train": { + "mean_centered_corr": -0.010898238979279995, + "mean_pairwise_disagreement": 0.026737251557793006, + "member_std": 0.030055008828639984 + }, + "val": { + "mean_centered_corr": -0.008359640836715698, + "mean_pairwise_disagreement": 0.09315727751030128, + "member_std": 0.06754488497972488 + }, + "test": { + "mean_centered_corr": -0.005901505704969168, + "mean_pairwise_disagreement": 0.05854524725492467, + "member_std": 0.06570815294981003 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..df76434d4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.9-greedy-heads-evaluation/0/report.json @@ -0,0 +1,256 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7904454637331882, + "recall": 0.9662742489219401, + "f1-score": 0.8695605693729521, + "support": 84179.0 + }, + "1": { + "precision": 0.5589560354202268, + "recall": 0.14299340275017883, + "f1-score": 0.22772872559258203, + "support": 25162.0 + }, + "accuracy": 0.776817479262125, + "macro avg": { + "precision": 0.6747007495767074, + "recall": 0.5546338258360595, + "f1-score": 0.5486446474827671, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7371741657277673, + "recall": 0.776817479262125, + "f1-score": 0.7218595893818996, + "support": 109341.0 + }, + "cross-entropy": 0.4989347278683794, + "roc-auc": 0.6834597818089504, + "score": 0.6834597818089504 + }, + "val": { + "0": { + "precision": 0.7030905642970632, + "recall": 0.8840056708338704, + "f1-score": 0.7832367466955951, + "support": 15518.0 + }, + "1": { + "precision": 0.6210526315789474, + "recall": 0.3374127873727554, + "f1-score": 0.43726376639739123, + "support": 8743.0 + }, + "accuracy": 0.6870285643625572, + "macro avg": { + "precision": 0.6620715979380053, + "recall": 0.6107092291033129, + "f1-score": 0.6102502565464931, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6735263399965609, + "recall": 0.6870285643625572, + "f1-score": 0.6585575592446575, + "support": 24261.0 + }, + "cross-entropy": 0.6200265526927166, + "roc-auc": 0.6506721330887921, + "score": 0.6506721330887921 + }, + "test": { + "0": { + "precision": 0.6765875561257216, + "recall": 0.9349958633731238, + "f1-score": 0.7850745528072047, + "support": 16922.0 + }, + "1": { + "precision": 0.6416938110749185, + "recall": 0.2066505821881884, + "f1-score": 0.312623978417837, + "support": 9533.0 + }, + "accuracy": 0.6725382725382726, + "macro avg": { + "precision": 0.6591406836003201, + "recall": 0.5708232227806561, + "f1-score": 0.5488492656125209, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6640136732465189, + "recall": 0.6725382725382726, + "f1-score": 0.6148280464509831, + "support": 26455.0 + }, + "cross-entropy": 0.6485088259427312, + "roc-auc": 0.6064667434006787, + "score": 0.6064667434006787 + } + }, + "time": "0:01:07.634554", + "diversity": { + "train": { + "mean_centered_corr": -0.010898238979279995, + "mean_pairwise_disagreement": 0.026737251557793006, + "member_std": 0.030055008828639984 + }, + "val": { + "mean_centered_corr": -0.008359640836715698, + "mean_pairwise_disagreement": 0.09315727751030128, + "member_std": 0.06754488497972488 + }, + "test": { + "mean_centered_corr": -0.005901505704969168, + "mean_pairwise_disagreement": 0.05854524725492467, + "member_std": 0.06570815294981003 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24, + 22 + ], + "head_selection_time": "0:00:15.957292" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9ee26893d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-best-head-evaluation/0/report.json @@ -0,0 +1,255 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9383522868156433, + "min_features_per_member": 98.0, + "mean_features_per_member": 103.21875, + "max_features_per_member": 107.0, + "min_members_per_feature": 25.0, + "mean_members_per_feature": 30.027273178100586, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8836170743012027, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7913176750815127, + "recall": 0.9629598831062379, + "f1-score": 0.8687418616739097, + "support": 84179.0 + }, + "1": { + "precision": 0.548312327973345, + "recall": 0.15042524441618313, + "f1-score": 0.2360829564946203, + "support": 25162.0 + }, + "accuracy": 0.77597607484841, + "macro avg": { + "precision": 0.6698150015274289, + "recall": 0.5566925637612106, + "f1-score": 0.552412409084265, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7353962865453213, + "recall": 0.77597607484841, + "f1-score": 0.7231517959883821, + "support": 109341.0 + }, + "cross-entropy": 0.5020020262037247, + "roc-auc": 0.677216119994803, + "score": 0.677216119994803 + }, + "val": { + "0": { + "precision": 0.7094869155334019, + "recall": 0.8438587446835932, + "f1-score": 0.7708609271523179, + "support": 15518.0 + }, + "1": { + "precision": 0.5825292901447278, + "recall": 0.3867093674939952, + "f1-score": 0.4648381109507115, + "support": 8743.0 + }, + "accuracy": 0.6791146284159763, + "macro avg": { + "precision": 0.6460081028390648, + "recall": 0.6152840560887942, + "f1-score": 0.6178495190515148, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6637348641433859, + "recall": 0.6791146284159763, + "f1-score": 0.6605786847859421, + "support": 24261.0 + }, + "cross-entropy": 0.6205532929394407, + "roc-auc": 0.6503189442353507, + "score": 0.6503189442353507 + }, + "test": { + "0": { + "precision": 0.6828174268330595, + "recall": 0.9085805460347477, + "f1-score": 0.779685083293187, + "support": 16922.0 + }, + "1": { + "precision": 0.6071609954291518, + "recall": 0.2508129654883038, + "f1-score": 0.35498478212456386, + "support": 9533.0 + }, + "accuracy": 0.6715554715554716, + "macro avg": { + "precision": 0.6449892111311056, + "recall": 0.5796967557615258, + "f1-score": 0.5673349327088755, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6555548012207573, + "recall": 0.6715554715554716, + "f1-score": 0.6266452809480543, + "support": 26455.0 + }, + "cross-entropy": 0.640466255418233, + "roc-auc": 0.6050413611236272, + "score": 0.6050413611236272 + } + }, + "time": "0:01:07.102406", + "diversity": { + "train": { + "mean_centered_corr": -0.012264390476047993, + "mean_pairwise_disagreement": 0.0252751933799292, + "member_std": 0.028376618400216103 + }, + "val": { + "mean_centered_corr": -0.007350938394665718, + "mean_pairwise_disagreement": 0.09498742505893569, + "member_std": 0.06712553650140762 + }, + "test": { + "mean_centered_corr": -0.0008085499866865575, + "mean_pairwise_disagreement": 0.06076119826119826, + "member_std": 0.06589463353157043 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24 + ], + "head_selection_time": "0:00:04.832161" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0.toml new file mode 100644 index 000000000..43b32f400 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0.toml @@ -0,0 +1,46 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0/report.json new file mode 100644 index 000000000..9bb1a7364 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-evaluation/0/report.json @@ -0,0 +1,251 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9383522868156433, + "min_features_per_member": 98.0, + "mean_features_per_member": 103.21875, + "max_features_per_member": 107.0, + "min_members_per_feature": 25.0, + "mean_members_per_feature": 30.027273178100586, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8836170743012027, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7858464296612756, + "recall": 0.9789496192637118, + "f1-score": 0.8718333924027867, + "support": 84179.0 + }, + "1": { + "precision": 0.6041992405628769, + "recall": 0.10750337810984818, + "f1-score": 0.18252977495866932, + "support": 25162.0 + }, + "accuracy": 0.7784088310880639, + "macro avg": { + "precision": 0.6950228351120762, + "recall": 0.54322649868678, + "f1-score": 0.527181583680728, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7440450324535136, + "recall": 0.7784088310880639, + "f1-score": 0.7132080128824888, + "support": 109341.0 + }, + "cross-entropy": 0.49252182617427764, + "roc-auc": 0.698478119380352, + "score": 0.698478119380352 + }, + "val": { + "0": { + "precision": 0.6768198008950589, + "recall": 0.9550844180951154, + "f1-score": 0.7922279238828309, + "support": 15518.0 + }, + "1": { + "precision": 0.7050359712230215, + "recall": 0.19055244195356286, + "f1-score": 0.30001800828381053, + "support": 8743.0 + }, + "accuracy": 0.6795680309962491, + "macro avg": { + "precision": 0.6909278860590402, + "recall": 0.5728184300243391, + "f1-score": 0.5461229660833207, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6869881359668769, + "recall": 0.6795680309962491, + "f1-score": 0.6148489497233884, + "support": 24261.0 + }, + "cross-entropy": 0.6755424825758085, + "roc-auc": 0.6252559501617828, + "score": 0.6252559501617828 + }, + "test": { + "0": { + "precision": 0.6574199189253637, + "recall": 0.9775440255288973, + "f1-score": 0.786142001710864, + "support": 16922.0 + }, + "1": { + "precision": 0.7061098221191029, + "recall": 0.09577257946082031, + "f1-score": 0.168668021429891, + "support": 9533.0 + }, + "accuracy": 0.6597996597996598, + "macro avg": { + "precision": 0.6817648705222332, + "recall": 0.5366583024948588, + "f1-score": 0.4774050115703775, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6749652164927767, + "recall": 0.6597996597996598, + "f1-score": 0.5636366358435982, + "support": 26455.0 + }, + "cross-entropy": 0.7385854201893112, + "roc-auc": 0.5929062493223765, + "score": 0.5929062493223765 + } + }, + "time": "0:01:07.102406", + "diversity": { + "train": { + "mean_centered_corr": -0.012264390476047993, + "mean_pairwise_disagreement": 0.0252751933799292, + "member_std": 0.028376618400216103 + }, + "val": { + "mean_centered_corr": -0.007350938394665718, + "mean_pairwise_disagreement": 0.09498742505893569, + "member_std": 0.06712553650140762 + }, + "test": { + "mean_centered_corr": -0.0008085499866865575, + "mean_pairwise_disagreement": 0.06076119826119826, + "member_std": 0.06589463353157043 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9495dd28d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/mfb_keep0.95-greedy-heads-evaluation/0/report.json @@ -0,0 +1,256 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1867072, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9383522868156433, + "min_features_per_member": 98.0, + "mean_features_per_member": 103.21875, + "max_features_per_member": 107.0, + "min_members_per_feature": 25.0, + "mean_members_per_feature": 30.027273178100586, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8836170743012027, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 1676 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 960 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.790086671260664, + "recall": 0.9670464130008672, + "f1-score": 0.8696557912954298, + "support": 84179.0 + }, + "1": { + "precision": 0.5602409638554217, + "recall": 0.14044988474684048, + "f1-score": 0.2245948522402288, + "support": 25162.0 + }, + "accuracy": 0.776826624962274, + "macro avg": { + "precision": 0.6751638175580428, + "recall": 0.5537481488738538, + "f1-score": 0.5471253217678294, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7371936330615374, + "recall": 0.776826624962274, + "f1-score": 0.7212117186373512, + "support": 109341.0 + }, + "cross-entropy": 0.4990311598658022, + "roc-auc": 0.6831394595121877, + "score": 0.6831394595121877 + }, + "val": { + "0": { + "precision": 0.7033659511245454, + "recall": 0.8847145250676633, + "f1-score": 0.7836858178497017, + "support": 15518.0 + }, + "1": { + "precision": 0.6227330240404892, + "recall": 0.33775591902093105, + "f1-score": 0.43796811271783465, + "support": 8743.0 + }, + "accuracy": 0.6876056221919954, + "macro avg": { + "precision": 0.6630494875825172, + "recall": 0.6112352220442971, + "f1-score": 0.6108269652837681, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6743080523777542, + "recall": 0.6876056221919954, + "f1-score": 0.6590986245778698, + "support": 24261.0 + }, + "cross-entropy": 0.6190109506021244, + "roc-auc": 0.6505655760961024, + "score": 0.6505655760961024 + }, + "test": { + "0": { + "precision": 0.6759120972903776, + "recall": 0.9360595674270181, + "f1-score": 0.7849939291820502, + "support": 16922.0 + }, + "1": { + "precision": 0.6417218543046358, + "recall": 0.2032938214622889, + "f1-score": 0.30877081175814547, + "support": 9533.0 + }, + "accuracy": 0.672009072009072, + "macro avg": { + "precision": 0.6588169757975066, + "recall": 0.5696766944446535, + "f1-score": 0.5468823704700978, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6635917198047199, + "recall": 0.672009072009072, + "f1-score": 0.6133879953925176, + "support": 26455.0 + }, + "cross-entropy": 0.6457100309018923, + "roc-auc": 0.6064442039882287, + "score": 0.6064442039882287 + } + }, + "time": "0:01:07.102406", + "diversity": { + "train": { + "mean_centered_corr": -0.012264390476047993, + "mean_pairwise_disagreement": 0.0252751933799292, + "member_std": 0.028376618400216103 + }, + "val": { + "mean_centered_corr": -0.007350938394665718, + "mean_pairwise_disagreement": 0.09498742505893569, + "member_std": 0.06712553650140762 + }, + "test": { + "mean_centered_corr": -0.0008085499866865575, + "mean_pairwise_disagreement": 0.06076119826119826, + "member_std": 0.06589463353157043 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24, + 22 + ], + "head_selection_time": "0:00:15.988939" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..eb9855f03 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,399 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7799945571081353, + "recall": 0.9873840268950689, + "f1-score": 0.8715214428017196, + "support": 84179.0 + }, + "1": { + "precision": 0.6179856115107913, + "recall": 0.0682775614021143, + "f1-score": 0.12296900722926062, + "support": 25162.0 + }, + "accuracy": 0.7758754721467702, + "macro avg": { + "precision": 0.6989900843094633, + "recall": 0.5278307941485916, + "f1-score": 0.4972452250154901, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7427123931520679, + "recall": 0.7758754721467702, + "f1-score": 0.6992614819098838, + "support": 109341.0 + }, + "cross-entropy": 0.4998011928817743, + "roc-auc": 0.6838112658195706, + "score": 0.6838112658195706 + }, + "val": { + "0": { + "precision": 0.6737931657928042, + "recall": 0.9606263693774971, + "f1-score": 0.7920408054832369, + "support": 15518.0 + }, + "1": { + "precision": 0.71408516612073, + "recall": 0.1745396317053643, + "f1-score": 0.28051470588235294, + "support": 8743.0 + }, + "accuracy": 0.6773422365112732, + "macro avg": { + "precision": 0.6939391659567671, + "recall": 0.5675830005414307, + "f1-score": 0.5362777556827949, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6883132992937752, + "recall": 0.6773422365112732, + "f1-score": 0.6077008075931859, + "support": 24261.0 + }, + "cross-entropy": 0.648962937315794, + "roc-auc": 0.6377017988002615, + "score": 0.6377017988002615 + }, + "test": { + "0": { + "precision": 0.6573357157037919, + "recall": 0.9783122562344877, + "f1-score": 0.7863300638848647, + "support": 16922.0 + }, + "1": { + "precision": 0.7110236220472441, + "recall": 0.09472359173397671, + "f1-score": 0.16717578450430437, + "support": 9533.0 + }, + "accuracy": 0.6599130599130599, + "macro avg": { + "precision": 0.684179668875518, + "recall": 0.5365179239842321, + "f1-score": 0.47675292419458454, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6766820325124152, + "recall": 0.6599130599130599, + "f1-score": 0.5632192060003484, + "support": 26455.0 + }, + "cross-entropy": 0.6873082211438988, + "roc-auc": 0.6003639247256524, + "score": 0.6003639247256524 + } + }, + "time": "0:01:59.726664", + "diversity": { + "train": { + "mean_centered_corr": -0.01748897135257721, + "mean_pairwise_disagreement": 0.024599776048355382, + "member_std": 0.027759695425629616 + }, + "val": { + "mean_centered_corr": -0.004482195246964693, + "mean_pairwise_disagreement": 0.06812864068310882, + "member_std": 0.05873022601008415 + }, + "test": { + "mean_centered_corr": 0.0009574671857990324, + "mean_pairwise_disagreement": 0.04505863578444223, + "member_std": 0.060155753046274185 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.870529" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..dc1b937df --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "ecom-offers" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..bbf83da26 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-evaluation/0/report.json @@ -0,0 +1,395 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7814940049542719, + "recall": 0.9856615070266931, + "f1-score": 0.8717835566062516, + "support": 84179.0 + }, + "1": { + "precision": 0.6192429022082019, + "recall": 0.07801446625864399, + "f1-score": 0.138571226881265, + "support": 25162.0 + }, + "accuracy": 0.7767900421616777, + "macro avg": { + "precision": 0.7003684535812369, + "recall": 0.5318379866426686, + "f1-score": 0.5051773917437583, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441561148005635, + "recall": 0.7767900421616777, + "f1-score": 0.7030537238761676, + "support": 109341.0 + }, + "cross-entropy": 0.49570326153659916, + "roc-auc": 0.6915131788984843, + "score": 0.6915131788984843 + }, + "val": { + "0": { + "precision": 0.6750283511000227, + "recall": 0.9589508957339863, + "f1-score": 0.7923222319836009, + "support": 15518.0 + }, + "1": { + "precision": 0.7125451263537906, + "recall": 0.18060162415646802, + "f1-score": 0.28816497855643763, + "support": 8743.0 + }, + "accuracy": 0.6784551337537612, + "macro avg": { + "precision": 0.6937867387269067, + "recall": 0.5697762599452272, + "f1-score": 0.5402436052700192, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6885483694852375, + "recall": 0.6784551337537612, + "f1-score": 0.6106377644549051, + "support": 24261.0 + }, + "cross-entropy": 0.6742112826823753, + "roc-auc": 0.6227441474841354, + "score": 0.6227441474841354 + }, + "test": { + "0": { + "precision": 0.657082804568528, + "recall": 0.9791395816097388, + "f1-score": 0.7864160615121742, + "support": 16922.0 + }, + "1": { + "precision": 0.7150928167877321, + "recall": 0.0929403125983426, + "f1-score": 0.16450055699962868, + "support": 9533.0 + }, + "accuracy": 0.6597996597996598, + "macro avg": { + "precision": 0.6860878106781301, + "recall": 0.5360399471040407, + "f1-score": 0.4754583092559015, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6779865825494644, + "recall": 0.6597996597996598, + "f1-score": 0.5623102023355311, + "support": 26455.0 + }, + "cross-entropy": 0.734778075293986, + "roc-auc": 0.5911932818714825, + "score": 0.5911932818714825 + } + }, + "time": "0:01:59.726664", + "diversity": { + "train": { + "mean_centered_corr": -0.01748897135257721, + "mean_pairwise_disagreement": 0.024599776048355382, + "member_std": 0.027759695425629616 + }, + "val": { + "mean_centered_corr": -0.004482195246964693, + "mean_pairwise_disagreement": 0.06812864068310882, + "member_std": 0.05873022601008415 + }, + "test": { + "mean_centered_corr": 0.0009574671857990324, + "mean_pairwise_disagreement": 0.04505863578444223, + "member_std": 0.060155753046274185 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8b014d4b0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,401 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "ecom-offers", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 110, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/ecom-offers" + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7815670375259287, + "recall": 0.9847230306846125, + "f1-score": 0.8714616876665668, + "support": 84179.0 + }, + "1": { + "precision": 0.6080463273392258, + "recall": 0.07928622526031318, + "f1-score": 0.1402805611222445, + "support": 25162.0 + }, + "accuracy": 0.7763601942546712, + "macro avg": { + "precision": 0.6948066824325773, + "recall": 0.5320046279724628, + "f1-score": 0.5058711243944056, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7416357390220023, + "recall": 0.7763601942546712, + "f1-score": 0.7031992837548755, + "support": 109341.0 + }, + "cross-entropy": 0.49883518991355796, + "roc-auc": 0.6843233749530935, + "score": 0.6843233749530935 + }, + "val": { + "0": { + "precision": 0.6819298326922185, + "recall": 0.9481891996391287, + "f1-score": 0.7933144628656154, + "support": 15518.0 + }, + "1": { + "precision": 0.7004470938897168, + "recall": 0.21502916619009493, + "f1-score": 0.3290452437210116, + "support": 8743.0 + }, + "accuracy": 0.6839784015498125, + "macro avg": { + "precision": 0.6911884632909677, + "recall": 0.5816091829146118, + "f1-score": 0.5611798532933134, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.688602946523047, + "recall": 0.6839784015498125, + "f1-score": 0.626004550537959, + "support": 24261.0 + }, + "cross-entropy": 0.6356156215085841, + "roc-auc": 0.6386263098818863, + "score": 0.6386263098818863 + }, + "test": { + "0": { + "precision": 0.6633519282076158, + "recall": 0.9697435291336721, + "f1-score": 0.7878060489678349, + "support": 16922.0 + }, + "1": { + "precision": 0.7018054746651136, + "recall": 0.12640302108465332, + "f1-score": 0.21422222222222223, + "support": 9533.0 + }, + "accuracy": 0.6658476658476659, + "macro avg": { + "precision": 0.6825787014363647, + "recall": 0.5480732751091627, + "f1-score": 0.5010141355950286, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6772085775510037, + "recall": 0.6658476658476659, + "f1-score": 0.5811164016283555, + "support": 26455.0 + }, + "cross-entropy": 0.6724107525253293, + "roc-auc": 0.6021922454924368, + "score": 0.6021922454924368 + } + }, + "time": "0:01:59.726664", + "diversity": { + "train": { + "mean_centered_corr": -0.01748897135257721, + "mean_pairwise_disagreement": 0.024599776048355382, + "member_std": 0.027759695425629616 + }, + "val": { + "mean_centered_corr": -0.004482195246964693, + "mean_pairwise_disagreement": 0.06812864068310882, + "member_std": 0.05873022601008415 + }, + "test": { + "mean_centered_corr": 0.0009574671857990324, + "mean_pairwise_disagreement": 0.04505863578444223, + "member_std": 0.060155753046274185 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11, + 17, + 14 + ], + "head_selection_time": "0:00:20.678561" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7d88db838 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-best-head-evaluation/0/report.json @@ -0,0 +1,238 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7795739735912358, + "recall": 0.9882037087634683, + "f1-score": 0.8715778003625201, + "support": 84179.0 + }, + "1": { + "precision": 0.623006833712984, + "recall": 0.06521739130434782, + "f1-score": 0.11807454309972658, + "support": 25162.0 + }, + "accuracy": 0.7758023065455776, + "macro avg": { + "precision": 0.7012904036521099, + "recall": 0.5267105500339081, + "f1-score": 0.4948261717311233, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7435441003175637, + "recall": 0.7758023065455776, + "f1-score": 0.6981785360495321, + "support": 109341.0 + }, + "cross-entropy": 0.5001891736479179, + "roc-auc": 0.6828250183964067, + "score": 0.6828250183964067 + }, + "val": { + "0": { + "precision": 0.6731150793650794, + "recall": 0.9619151952571208, + "f1-score": 0.7920093383562371, + "support": 15518.0 + }, + "1": { + "precision": 0.7165467625899281, + "recall": 0.17087956079149033, + "f1-score": 0.2759512375323236, + "support": 8743.0 + }, + "accuracy": 0.6768476155146119, + "macro avg": { + "precision": 0.6948309209775038, + "recall": 0.5663973780243056, + "f1-score": 0.5339802879442803, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6887666686002656, + "recall": 0.6768476155146119, + "f1-score": 0.6060361313365975, + "support": 24261.0 + }, + "cross-entropy": 0.6456461560255325, + "roc-auc": 0.6394659925462143, + "score": 0.6394659925462143 + }, + "test": { + "0": { + "precision": 0.6568744053282588, + "recall": 0.9791395816097388, + "f1-score": 0.78626678688369, + "support": 16922.0 + }, + "1": { + "precision": 0.7132412672623883, + "recall": 0.09210112241686773, + "f1-score": 0.16313638052768487, + "support": 9533.0 + }, + "accuracy": 0.6594972594972595, + "macro avg": { + "precision": 0.6850578362953236, + "recall": 0.5356203520133033, + "f1-score": 0.47470158370568744, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6771860777840538, + "recall": 0.6594972594972595, + "f1-score": 0.5617231405487138, + "support": 26455.0 + }, + "cross-entropy": 0.6814826293175997, + "roc-auc": 0.601268355843962, + "score": 0.601268355843962 + } + }, + "time": "0:02:55.716058", + "diversity": { + "train": { + "mean_centered_corr": -0.01672052964568138, + "mean_pairwise_disagreement": 0.02467565216955184, + "member_std": 0.02795952372252941 + }, + "val": { + "mean_centered_corr": 0.002118270844221115, + "mean_pairwise_disagreement": 0.06719424577610954, + "member_std": 0.05840356647968292 + }, + "test": { + "mean_centered_corr": 0.008818755857646465, + "mean_pairwise_disagreement": 0.04434455039293749, + "member_std": 0.05988753214478493 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.867127" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0.toml new file mode 100644 index 000000000..5b186ba93 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0/report.json new file mode 100644 index 000000000..5f2e1b197 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/0/report.json @@ -0,0 +1,234 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7813565066752028, + "recall": 0.9858872165266872, + "f1-score": 0.8717862526458431, + "support": 84179.0 + }, + "1": { + "precision": 0.6200831467860569, + "recall": 0.0770606470073921, + "f1-score": 0.13708508607585987, + "support": 25162.0 + }, + "accuracy": 0.7767443136609323, + "macro avg": { + "precision": 0.7007198267306298, + "recall": 0.5314739317670396, + "f1-score": 0.5044356693608515, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7442436187234676, + "recall": 0.7767443136609323, + "f1-score": 0.7027138026661107, + "support": 109341.0 + }, + "cross-entropy": 0.4956990087527342, + "roc-auc": 0.6915444543929165, + "score": 0.6915444543929165 + }, + "val": { + "0": { + "precision": 0.6749523723124377, + "recall": 0.9588864544400052, + "f1-score": 0.7922478969225855, + "support": 15518.0 + }, + "1": { + "precision": 0.7119638826185102, + "recall": 0.1803728697243509, + "f1-score": 0.2878262456652674, + "support": 8743.0 + }, + "accuracy": 0.6783314785045959, + "macro avg": { + "precision": 0.6934581274654739, + "recall": 0.569629662082178, + "f1-score": 0.5400370712939264, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6882903070474441, + "recall": 0.6783314785045959, + "f1-score": 0.6104681476565729, + "support": 24261.0 + }, + "cross-entropy": 0.6752158783074748, + "roc-auc": 0.6225317005394864, + "score": 0.6225317005394864 + }, + "test": { + "0": { + "precision": 0.6569933396764985, + "recall": 0.9793168656187212, + "f1-score": 0.7864091491481991, + "support": 16922.0 + }, + "1": { + "precision": 0.7156783103168156, + "recall": 0.0924158187349208, + "f1-score": 0.16369379412857674, + "support": 9533.0 + }, + "accuracy": 0.6597240597240597, + "macro avg": { + "precision": 0.6863358249966571, + "recall": 0.535866342176821, + "f1-score": 0.47505147163838796, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6781403374128109, + "recall": 0.6597240597240597, + "f1-score": 0.5620150656327177, + "support": 26455.0 + }, + "cross-entropy": 0.7364672294386888, + "roc-auc": 0.5910259812848737, + "score": 0.5910259812848737 + } + }, + "time": "0:02:55.716058", + "diversity": { + "train": { + "mean_centered_corr": -0.01672052964568138, + "mean_pairwise_disagreement": 0.02467565216955184, + "member_std": 0.02795952372252941 + }, + "val": { + "mean_centered_corr": 0.002118270844221115, + "mean_pairwise_disagreement": 0.06719424577610954, + "member_std": 0.05840356647968292 + }, + "test": { + "mean_centered_corr": 0.008818755857646465, + "mean_pairwise_disagreement": 0.04434455039293749, + "member_std": 0.05988753214478493 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1605df310 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-greedy-heads-evaluation/0/report.json @@ -0,0 +1,238 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7795739735912358, + "recall": 0.9882037087634683, + "f1-score": 0.8715778003625201, + "support": 84179.0 + }, + "1": { + "precision": 0.623006833712984, + "recall": 0.06521739130434782, + "f1-score": 0.11807454309972658, + "support": 25162.0 + }, + "accuracy": 0.7758023065455776, + "macro avg": { + "precision": 0.7012904036521099, + "recall": 0.5267105500339081, + "f1-score": 0.4948261717311233, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7435441003175637, + "recall": 0.7758023065455776, + "f1-score": 0.6981785360495321, + "support": 109341.0 + }, + "cross-entropy": 0.5001891736479179, + "roc-auc": 0.6828250183964067, + "score": 0.6828250183964067 + }, + "val": { + "0": { + "precision": 0.6731150793650794, + "recall": 0.9619151952571208, + "f1-score": 0.7920093383562371, + "support": 15518.0 + }, + "1": { + "precision": 0.7165467625899281, + "recall": 0.17087956079149033, + "f1-score": 0.2759512375323236, + "support": 8743.0 + }, + "accuracy": 0.6768476155146119, + "macro avg": { + "precision": 0.6948309209775038, + "recall": 0.5663973780243056, + "f1-score": 0.5339802879442803, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6887666686002656, + "recall": 0.6768476155146119, + "f1-score": 0.6060361313365975, + "support": 24261.0 + }, + "cross-entropy": 0.6456461560255325, + "roc-auc": 0.6394659925462143, + "score": 0.6394659925462143 + }, + "test": { + "0": { + "precision": 0.6568744053282588, + "recall": 0.9791395816097388, + "f1-score": 0.78626678688369, + "support": 16922.0 + }, + "1": { + "precision": 0.7132412672623883, + "recall": 0.09210112241686773, + "f1-score": 0.16313638052768487, + "support": 9533.0 + }, + "accuracy": 0.6594972594972595, + "macro avg": { + "precision": 0.6850578362953236, + "recall": 0.5356203520133033, + "f1-score": 0.47470158370568744, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6771860777840538, + "recall": 0.6594972594972595, + "f1-score": 0.5617231405487138, + "support": 26455.0 + }, + "cross-entropy": 0.6814826293175997, + "roc-auc": 0.601268355843962, + "score": 0.601268355843962 + } + }, + "time": "0:02:55.716058", + "diversity": { + "train": { + "mean_centered_corr": -0.01672052964568138, + "mean_pairwise_disagreement": 0.02467565216955184, + "member_std": 0.02795952372252941 + }, + "val": { + "mean_centered_corr": 0.002118270844221115, + "mean_pairwise_disagreement": 0.06719424577610954, + "member_std": 0.05840356647968292 + }, + "test": { + "mean_centered_corr": 0.008818755857646465, + "mean_pairwise_disagreement": 0.04434455039293749, + "member_std": 0.05988753214478493 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.170047" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7d6ee9e87 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,264 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7920123669380088, + "recall": 0.9616412644483778, + "f1-score": 0.86862281167678, + "support": 84179.0 + }, + "1": { + "precision": 0.547315295107248, + "recall": 0.15515459820364041, + "f1-score": 0.2417711720080508, + "support": 25162.0 + }, + "accuracy": 0.7760492404496027, + "macro avg": { + "precision": 0.6696638310226284, + "recall": 0.5583979313260091, + "f1-score": 0.5551969918424154, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7357016717604853, + "recall": 0.7760492404496027, + "f1-score": 0.724369137781859, + "support": 109341.0 + }, + "cross-entropy": 0.5019376417142178, + "roc-auc": 0.677710781278526, + "score": 0.677710781278526 + }, + "val": { + "0": { + "precision": 0.7096030308021742, + "recall": 0.8328392834128109, + "f1-score": 0.7662980640953425, + "support": 15518.0 + }, + "1": { + "precision": 0.5710978835978836, + "recall": 0.39505890426627016, + "f1-score": 0.46704076803461564, + "support": 8743.0 + }, + "accuracy": 0.6750752236099089, + "macro avg": { + "precision": 0.6403504572000289, + "recall": 0.6139490938395405, + "f1-score": 0.616669416064979, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6596895687846518, + "recall": 0.6750752236099089, + "f1-score": 0.6584539299104807, + "support": 24261.0 + }, + "cross-entropy": 0.6233683660120171, + "roc-auc": 0.6500073735640511, + "score": 0.6500073735640511 + }, + "test": { + "0": { + "precision": 0.6827712348322627, + "recall": 0.9044439191584919, + "f1-score": 0.7781280187096447, + "support": 16922.0 + }, + "1": { + "precision": 0.5996533795493935, + "recall": 0.2540648274415189, + "f1-score": 0.3569112879457707, + "support": 9533.0 + }, + "accuracy": 0.67008127008127, + "macro avg": { + "precision": 0.641212307190828, + "recall": 0.5792543733000054, + "f1-score": 0.5675196533277077, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6528199018361716, + "recall": 0.67008127008127, + "f1-score": 0.6263435131578771, + "support": 26455.0 + }, + "cross-entropy": 0.6442461271803751, + "roc-auc": 0.6048362127969982, + "score": 0.6048362127969982 + } + }, + "time": "0:02:06.129189", + "diversity": { + "train": { + "mean_centered_corr": -0.010998313315212727, + "mean_pairwise_disagreement": 0.026646919329909303, + "member_std": 0.02979286201298237 + }, + "val": { + "mean_centered_corr": -0.007380904629826546, + "mean_pairwise_disagreement": 0.09225953042916349, + "member_std": 0.0662473514676094 + }, + "test": { + "mean_centered_corr": -0.004791198763996363, + "mean_pairwise_disagreement": 0.057417952579242905, + "member_std": 0.06390407681465149 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 24 + ], + "head_selection_time": "0:00:04.829893" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0.toml new file mode 100644 index 000000000..957ed8e36 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0.toml @@ -0,0 +1,53 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0/report.json new file mode 100644 index 000000000..505364548 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-evaluation/0/report.json @@ -0,0 +1,260 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7843980518925693, + "recall": 0.9815155798952233, + "f1-score": 0.8719553379205538, + "support": 84179.0 + }, + "1": { + "precision": 0.6117764471057884, + "recall": 0.0974485335029012, + "f1-score": 0.1681179293794995, + "support": 25162.0 + }, + "accuracy": 0.7780704401825482, + "macro avg": { + "precision": 0.6980872494991788, + "recall": 0.5394820566990622, + "f1-score": 0.5200366336500266, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7446736592160346, + "recall": 0.7780704401825482, + "f1-score": 0.7099853827005539, + "support": 109341.0 + }, + "cross-entropy": 0.4926461998052048, + "roc-auc": 0.6983951997801772, + "score": 0.6983951997801772 + }, + "val": { + "0": { + "precision": 0.6763377583139455, + "recall": 0.9554066245650212, + "f1-score": 0.7920083335559176, + "support": 15518.0 + }, + "1": { + "precision": 0.7042735042735043, + "recall": 0.18849365206450874, + "f1-score": 0.29739240277903095, + "support": 8743.0 + }, + "accuracy": 0.6790321915831994, + "macro avg": { + "precision": 0.6903056312937249, + "recall": 0.5719501383147649, + "f1-score": 0.5447003681674742, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6864050361229568, + "recall": 0.6790321915831994, + "f1-score": 0.6137622974163388, + "support": 24261.0 + }, + "cross-entropy": 0.6773464354872277, + "roc-auc": 0.6244884295114916, + "score": 0.6244884295114916 + }, + "test": { + "0": { + "precision": 0.6572257090649083, + "recall": 0.9777213095378797, + "f1-score": 0.7860604332953249, + "support": 16922.0 + }, + "1": { + "precision": 0.7056986729117877, + "recall": 0.09482849050666108, + "f1-score": 0.1671906787497688, + "support": 9533.0 + }, + "accuracy": 0.6595728595728596, + "macro avg": { + "precision": 0.681462190988348, + "recall": 0.5362749000222704, + "f1-score": 0.47662555602254686, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6746928330245492, + "recall": 0.6595728595728596, + "f1-score": 0.5630521032978657, + "support": 26455.0 + }, + "cross-entropy": 0.7412153243698456, + "roc-auc": 0.5923593741199417, + "score": 0.5923593741199417 + } + }, + "time": "0:02:06.129189", + "diversity": { + "train": { + "mean_centered_corr": -0.010998313315212727, + "mean_pairwise_disagreement": 0.026646919329909303, + "member_std": 0.02979286201298237 + }, + "val": { + "mean_centered_corr": -0.007380904629826546, + "mean_pairwise_disagreement": 0.09225953042916349, + "member_std": 0.0662473514676094 + }, + "test": { + "mean_centered_corr": -0.004791198763996363, + "mean_pairwise_disagreement": 0.057417952579242905, + "member_std": 0.06390407681465149 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c414f631d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,265 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.887499988079071, + "min_features_per_member": 92.0, + "mean_features_per_member": 97.625, + "max_features_per_member": 103.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.399999618530273, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.797659951359852, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 321, + "metrics": { + "train": { + "0": { + "precision": 0.7905577480122087, + "recall": 0.9661673338956271, + "f1-score": 0.8695852066482409, + "support": 84179.0 + }, + "1": { + "precision": 0.5593377688380009, + "recall": 0.1436690247198156, + "f1-score": 0.22861660079051382, + "support": 25162.0 + }, + "accuracy": 0.7768906448633175, + "macro avg": { + "precision": 0.6749477584251048, + "recall": 0.5549181793077214, + "f1-score": 0.5491009037193774, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.737348456749266, + "recall": 0.7768906448633175, + "f1-score": 0.7220828785133955, + "support": 109341.0 + }, + "cross-entropy": 0.4989600660711784, + "roc-auc": 0.6833803969604821, + "score": 0.6833803969604821 + }, + "val": { + "0": { + "precision": 0.7033559113300493, + "recall": 0.8832968166000773, + "f1-score": 0.7831228932183054, + "support": 15518.0 + }, + "1": { + "precision": 0.6205740624345275, + "recall": 0.3387853139654581, + "f1-score": 0.4382953536549275, + "support": 8743.0 + }, + "accuracy": 0.6870697827789456, + "macro avg": { + "precision": 0.6619649868822883, + "recall": 0.6110410652827677, + "f1-score": 0.6107091234366164, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6735236000117382, + "recall": 0.6870697827789456, + "f1-score": 0.6588564912397137, + "support": 24261.0 + }, + "cross-entropy": 0.6201426085246959, + "roc-auc": 0.6505360715210358, + "score": 0.6505360715210358 + }, + "test": { + "0": { + "precision": 0.6765422598435296, + "recall": 0.9351731473821061, + "f1-score": 0.7851065413142163, + "support": 16922.0 + }, + "1": { + "precision": 0.6419712793733682, + "recall": 0.20633588587013532, + "f1-score": 0.3122965785504485, + "support": 9533.0 + }, + "accuracy": 0.6725382725382726, + "macro avg": { + "precision": 0.6592567696084488, + "recall": 0.5707545166261208, + "f1-score": 0.5487015599323324, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6640846844580808, + "recall": 0.6725382725382726, + "f1-score": 0.6147305301621846, + "support": 26455.0 + }, + "cross-entropy": 0.648405879749753, + "roc-auc": 0.6064993654188358, + "score": 0.6064993654188358 + } + }, + "time": "0:02:06.129189", + "diversity": { + "train": { + "mean_centered_corr": -0.010998313315212727, + "mean_pairwise_disagreement": 0.026646919329909303, + "member_std": 0.02979286201298237 + }, + "val": { + "mean_centered_corr": -0.007380904629826546, + "mean_pairwise_disagreement": 0.09225953042916349, + "member_std": 0.0662473514676094 + }, + "test": { + "mean_centered_corr": -0.004791198763996363, + "mean_pairwise_disagreement": 0.057417952579242905, + "member_std": 0.06390407681465149 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 24, + 22 + ], + "head_selection_time": "0:00:15.977531" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..16776bce5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.779700580278046, + "recall": 0.9880492759476829, + "f1-score": 0.8715968394357931, + "support": 84179.0 + }, + "1": { + "precision": 0.6229385307346327, + "recall": 0.06605198314919322, + "f1-score": 0.11943945382680561, + "support": 25162.0 + }, + "accuracy": 0.7758754721467702, + "macro avg": { + "precision": 0.7013195555063394, + "recall": 0.5270506295484381, + "f1-score": 0.4955181466312994, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436258535917035, + "recall": 0.7758754721467702, + "f1-score": 0.6985072926354772, + "support": 109341.0 + }, + "cross-entropy": 0.5001462491291722, + "roc-auc": 0.6828783854988578, + "score": 0.6828783854988578 + }, + "val": { + "0": { + "precision": 0.6734214920792526, + "recall": 0.9615285474932337, + "f1-score": 0.7920902455209025, + "support": 15518.0 + }, + "1": { + "precision": 0.716254752851711, + "recall": 0.17236646460025162, + "f1-score": 0.2778648474232507, + "support": 8743.0 + }, + "accuracy": 0.677136144429331, + "macro avg": { + "precision": 0.6948381224654818, + "recall": 0.5669475060467426, + "f1-score": 0.5349775464720765, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.688857426250705, + "recall": 0.677136144429331, + "f1-score": 0.6067774943742981, + "support": 24261.0 + }, + "cross-entropy": 0.6457686779603163, + "roc-auc": 0.6396541901648656, + "score": 0.6396541901648656 + }, + "test": { + "0": { + "precision": 0.6569502280388657, + "recall": 0.9789032029310956, + "f1-score": 0.7862448679307972, + "support": 16922.0 + }, + "1": { + "precision": 0.7120967741935483, + "recall": 0.09262561628028952, + "f1-score": 0.16392833936693585, + "support": 9533.0 + }, + "accuracy": 0.6595350595350595, + "macro avg": { + "precision": 0.684523501116207, + "recall": 0.5357644096056925, + "f1-score": 0.4750866036488665, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.67682216243662, + "recall": 0.6595350595350595, + "f1-score": 0.5619945006354167, + "support": 26455.0 + }, + "cross-entropy": 0.6822343746621596, + "roc-auc": 0.6015418321886687, + "score": 0.6015418321886687 + } + }, + "time": "0:01:18.218817", + "diversity": { + "train": { + "mean_centered_corr": -0.016645358875393867, + "mean_pairwise_disagreement": 0.024673918911862297, + "member_std": 0.027964944019913673 + }, + "val": { + "mean_centered_corr": 0.002000282984226942, + "mean_pairwise_disagreement": 0.06748003233651247, + "member_std": 0.05873498693108559 + }, + "test": { + "mean_centered_corr": 0.008286470547318459, + "mean_pairwise_disagreement": 0.04453758969888002, + "member_std": 0.06023459881544113 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.887253" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..2990735ec --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..1b45b9b2d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7813988964010622, + "recall": 0.9857921809477423, + "f1-score": 0.8717754770796893, + "support": 84179.0 + }, + "1": { + "precision": 0.6194718421889914, + "recall": 0.0773785867578094, + "f1-score": 0.1375728669846317, + "support": 25162.0 + }, + "accuracy": 0.7767443136609323, + "macro avg": { + "precision": 0.7004353692950268, + "recall": 0.5315853838527759, + "f1-score": 0.5046741720321605, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.744135577626914, + "recall": 0.7767443136609323, + "f1-score": 0.7028177569636135, + "support": 109341.0 + }, + "cross-entropy": 0.4956556906392919, + "roc-auc": 0.6916462356019382, + "score": 0.6916462356019382 + }, + "val": { + "0": { + "precision": 0.6749512449544197, + "recall": 0.9590153370279675, + "f1-score": 0.7922911065562861, + "support": 15518.0 + }, + "1": { + "precision": 0.7124773960216998, + "recall": 0.18025849250829235, + "f1-score": 0.2877225011410315, + "support": 8743.0 + }, + "accuracy": 0.6783726969209843, + "macro avg": { + "precision": 0.6937143204880598, + "recall": 0.5696369147681299, + "f1-score": 0.5400068038486587, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6884746421260627, + "recall": 0.6783726969209843, + "f1-score": 0.6104583990361686, + "support": 24261.0 + }, + "cross-entropy": 0.6750056416383268, + "roc-auc": 0.6227001707049361, + "score": 0.6227001707049361 + }, + "test": { + "0": { + "precision": 0.6569412510901451, + "recall": 0.9793168656187212, + "f1-score": 0.7863718325899213, + "support": 16922.0 + }, + "1": { + "precision": 0.7152156224572823, + "recall": 0.09220602118955208, + "f1-score": 0.163352536703215, + "support": 9533.0 + }, + "accuracy": 0.6596484596484596, + "macro avg": { + "precision": 0.6860784367737137, + "recall": 0.5357614434041367, + "f1-score": 0.47486218464656815, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6779402902979667, + "recall": 0.6596484596484596, + "f1-score": 0.561868224663708, + "support": 26455.0 + }, + "cross-entropy": 0.7361612281771354, + "roc-auc": 0.5911765632808944, + "score": 0.5911765632808944 + } + }, + "time": "0:01:18.218817", + "diversity": { + "train": { + "mean_centered_corr": -0.016645358875393867, + "mean_pairwise_disagreement": 0.024673918911862297, + "member_std": 0.027964944019913673 + }, + "val": { + "mean_centered_corr": 0.002000282984226942, + "mean_pairwise_disagreement": 0.06748003233651247, + "member_std": 0.05873498693108559 + }, + "test": { + "mean_centered_corr": 0.008286470547318459, + "mean_pairwise_disagreement": 0.04453758969888002, + "member_std": 0.06023459881544113 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ec841c0fe --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.779700580278046, + "recall": 0.9880492759476829, + "f1-score": 0.8715968394357931, + "support": 84179.0 + }, + "1": { + "precision": 0.6229385307346327, + "recall": 0.06605198314919322, + "f1-score": 0.11943945382680561, + "support": 25162.0 + }, + "accuracy": 0.7758754721467702, + "macro avg": { + "precision": 0.7013195555063394, + "recall": 0.5270506295484381, + "f1-score": 0.4955181466312994, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436258535917035, + "recall": 0.7758754721467702, + "f1-score": 0.6985072926354772, + "support": 109341.0 + }, + "cross-entropy": 0.5001462491291722, + "roc-auc": 0.6828783854988578, + "score": 0.6828783854988578 + }, + "val": { + "0": { + "precision": 0.6734214920792526, + "recall": 0.9615285474932337, + "f1-score": 0.7920902455209025, + "support": 15518.0 + }, + "1": { + "precision": 0.716254752851711, + "recall": 0.17236646460025162, + "f1-score": 0.2778648474232507, + "support": 8743.0 + }, + "accuracy": 0.677136144429331, + "macro avg": { + "precision": 0.6948381224654818, + "recall": 0.5669475060467426, + "f1-score": 0.5349775464720765, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.688857426250705, + "recall": 0.677136144429331, + "f1-score": 0.6067774943742981, + "support": 24261.0 + }, + "cross-entropy": 0.6457686779603163, + "roc-auc": 0.6396541901648656, + "score": 0.6396541901648656 + }, + "test": { + "0": { + "precision": 0.6569502280388657, + "recall": 0.9789032029310956, + "f1-score": 0.7862448679307972, + "support": 16922.0 + }, + "1": { + "precision": 0.7120967741935483, + "recall": 0.09262561628028952, + "f1-score": 0.16392833936693585, + "support": 9533.0 + }, + "accuracy": 0.6595350595350595, + "macro avg": { + "precision": 0.684523501116207, + "recall": 0.5357644096056925, + "f1-score": 0.4750866036488665, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.67682216243662, + "recall": 0.6595350595350595, + "f1-score": 0.5619945006354167, + "support": 26455.0 + }, + "cross-entropy": 0.6822343746621596, + "roc-auc": 0.6015418321886687, + "score": 0.6015418321886687 + } + }, + "time": "0:01:18.218817", + "diversity": { + "train": { + "mean_centered_corr": -0.016645358875393867, + "mean_pairwise_disagreement": 0.024673918911862297, + "member_std": 0.027964944019913673 + }, + "val": { + "mean_centered_corr": 0.002000282984226942, + "mean_pairwise_disagreement": 0.06748003233651247, + "member_std": 0.05873498693108559 + }, + "test": { + "mean_centered_corr": 0.008286470547318459, + "mean_pairwise_disagreement": 0.04453758969888002, + "member_std": 0.06023459881544113 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.224192" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..090a927fe --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7796022418414591, + "recall": 0.9881561909739959, + "f1-score": 0.871576984131142, + "support": 84179.0 + }, + "1": { + "precision": 0.6227771471812334, + "recall": 0.06541610364835863, + "f1-score": 0.11839597194749146, + "support": 25162.0 + }, + "accuracy": 0.7758114522457267, + "macro avg": { + "precision": 0.7011896945113463, + "recall": 0.5267861473111772, + "f1-score": 0.4949864780393167, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7435130069539001, + "recall": 0.7758114522457267, + "f1-score": 0.6982518761792758, + "support": 109341.0 + }, + "cross-entropy": 0.5001282298547604, + "roc-auc": 0.6829040213009548, + "score": 0.6829040213009548 + }, + "val": { + "0": { + "precision": 0.6732235506429055, + "recall": 0.9615929887872149, + "f1-score": 0.7919751612132792, + "support": 15518.0 + }, + "1": { + "precision": 0.7156488549618321, + "recall": 0.1715658240878417, + "f1-score": 0.2767783005812344, + "support": 8743.0 + }, + "accuracy": 0.6768888339310004, + "macro avg": { + "precision": 0.6944362028023687, + "recall": 0.5665794064375282, + "f1-score": 0.5343767308972568, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6885124684806028, + "recall": 0.6768888339310004, + "f1-score": 0.6063123215732822, + "support": 24261.0 + }, + "cross-entropy": 0.6459156883389179, + "roc-auc": 0.6395828610304147, + "score": 0.6395828610304147 + }, + "test": { + "0": { + "precision": 0.6568199841395718, + "recall": 0.9789032029310956, + "f1-score": 0.7861515827440558, + "support": 16922.0 + }, + "1": { + "precision": 0.7109311740890688, + "recall": 0.09210112241686773, + "f1-score": 0.16307578008915305, + "support": 9533.0 + }, + "accuracy": 0.6593460593460594, + "macro avg": { + "precision": 0.6838755791143203, + "recall": 0.5355021626739817, + "f1-score": 0.4746136814166044, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6763188302476253, + "recall": 0.6593460593460594, + "f1-score": 0.5616276127304785, + "support": 26455.0 + }, + "cross-entropy": 0.6823614555900946, + "roc-auc": 0.601475038412775, + "score": 0.601475038412775 + } + }, + "time": "0:01:18.777407", + "diversity": { + "train": { + "mean_centered_corr": -0.01685778796672821, + "mean_pairwise_disagreement": 0.024678897417991834, + "member_std": 0.027964096516370773 + }, + "val": { + "mean_centered_corr": 0.0022963290102779865, + "mean_pairwise_disagreement": 0.06754152755451136, + "member_std": 0.05871695652604103 + }, + "test": { + "mean_centered_corr": 0.00823600497096777, + "mean_pairwise_disagreement": 0.04456837843934618, + "member_std": 0.060229621827602386 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.878729" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..3684da014 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..5037b8574 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7813841807909605, + "recall": 0.9857921809477423, + "f1-score": 0.8717663187641495, + "support": 84179.0 + }, + "1": { + "precision": 0.6192295447309774, + "recall": 0.07729910182020507, + "f1-score": 0.137441260643748, + "support": 25162.0 + }, + "accuracy": 0.7767260222606341, + "macro avg": { + "precision": 0.700306862760969, + "recall": 0.5315456413839738, + "f1-score": 0.5046037897039487, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7440684899472578, + "recall": 0.7767260222606341, + "f1-score": 0.702780420405569, + "support": 109341.0 + }, + "cross-entropy": 0.495653101339186, + "roc-auc": 0.6916612924072583, + "score": 0.6916612924072583 + }, + "val": { + "0": { + "precision": 0.67487416678003, + "recall": 0.9590797783219487, + "f1-score": 0.7922599877565143, + "support": 15518.0 + }, + "1": { + "precision": 0.7124094202898551, + "recall": 0.17991536086011667, + "f1-score": 0.28727970048397405, + "support": 8743.0 + }, + "accuracy": 0.6782902600882074, + "macro avg": { + "precision": 0.6936417935349426, + "recall": 0.5694975695910327, + "f1-score": 0.5397698441202442, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6884008442226911, + "recall": 0.6782902600882074, + "f1-score": 0.6102789213691511, + "support": 24261.0 + }, + "cross-entropy": 0.6751155148637754, + "roc-auc": 0.6226233688882503, + "score": 0.6226233688882503 + }, + "test": { + "0": { + "precision": 0.656953694893752, + "recall": 0.9792577709490604, + "f1-score": 0.7863616950600294, + "support": 16922.0 + }, + "1": { + "precision": 0.7148659626320065, + "recall": 0.09231091996223645, + "f1-score": 0.1635079895949461, + "support": 9533.0 + }, + "accuracy": 0.6596484596484596, + "macro avg": { + "precision": 0.6859098287628793, + "recall": 0.5357843454556485, + "f1-score": 0.4749348423274877, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6778222508698919, + "recall": 0.6596484596484596, + "f1-score": 0.561917757271383, + "support": 26455.0 + }, + "cross-entropy": 0.7362984524384457, + "roc-auc": 0.5911217830862241, + "score": 0.5911217830862241 + } + }, + "time": "0:01:18.777407", + "diversity": { + "train": { + "mean_centered_corr": -0.01685778796672821, + "mean_pairwise_disagreement": 0.024678897417991834, + "member_std": 0.027964096516370773 + }, + "val": { + "mean_centered_corr": 0.0022963290102779865, + "mean_pairwise_disagreement": 0.06754152755451136, + "member_std": 0.05871695652604103 + }, + "test": { + "mean_centered_corr": 0.00823600497096777, + "mean_pairwise_disagreement": 0.04456837843934618, + "member_std": 0.060229621827602386 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..095996c3d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7796022418414591, + "recall": 0.9881561909739959, + "f1-score": 0.871576984131142, + "support": 84179.0 + }, + "1": { + "precision": 0.6227771471812334, + "recall": 0.06541610364835863, + "f1-score": 0.11839597194749146, + "support": 25162.0 + }, + "accuracy": 0.7758114522457267, + "macro avg": { + "precision": 0.7011896945113463, + "recall": 0.5267861473111772, + "f1-score": 0.4949864780393167, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7435130069539001, + "recall": 0.7758114522457267, + "f1-score": 0.6982518761792758, + "support": 109341.0 + }, + "cross-entropy": 0.5001282298547604, + "roc-auc": 0.6829040213009548, + "score": 0.6829040213009548 + }, + "val": { + "0": { + "precision": 0.6732235506429055, + "recall": 0.9615929887872149, + "f1-score": 0.7919751612132792, + "support": 15518.0 + }, + "1": { + "precision": 0.7156488549618321, + "recall": 0.1715658240878417, + "f1-score": 0.2767783005812344, + "support": 8743.0 + }, + "accuracy": 0.6768888339310004, + "macro avg": { + "precision": 0.6944362028023687, + "recall": 0.5665794064375282, + "f1-score": 0.5343767308972568, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6885124684806028, + "recall": 0.6768888339310004, + "f1-score": 0.6063123215732822, + "support": 24261.0 + }, + "cross-entropy": 0.6459156883389179, + "roc-auc": 0.6395828610304147, + "score": 0.6395828610304147 + }, + "test": { + "0": { + "precision": 0.6568199841395718, + "recall": 0.9789032029310956, + "f1-score": 0.7861515827440558, + "support": 16922.0 + }, + "1": { + "precision": 0.7109311740890688, + "recall": 0.09210112241686773, + "f1-score": 0.16307578008915305, + "support": 9533.0 + }, + "accuracy": 0.6593460593460594, + "macro avg": { + "precision": 0.6838755791143203, + "recall": 0.5355021626739817, + "f1-score": 0.4746136814166044, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6763188302476253, + "recall": 0.6593460593460594, + "f1-score": 0.5616276127304785, + "support": 26455.0 + }, + "cross-entropy": 0.6823614555900946, + "roc-auc": 0.601475038412775, + "score": 0.601475038412775 + } + }, + "time": "0:01:18.777407", + "diversity": { + "train": { + "mean_centered_corr": -0.01685778796672821, + "mean_pairwise_disagreement": 0.024678897417991834, + "member_std": 0.027964096516370773 + }, + "val": { + "mean_centered_corr": 0.0022963290102779865, + "mean_pairwise_disagreement": 0.06754152755451136, + "member_std": 0.05871695652604103 + }, + "test": { + "mean_centered_corr": 0.00823600497096777, + "mean_pairwise_disagreement": 0.04456837843934618, + "member_std": 0.060229621827602386 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.169700" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e1fe7d509 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.779683735927936, + "recall": 0.9881205526318916, + "f1-score": 0.8716140457503327, + "support": 84179.0 + }, + "1": { + "precision": 0.6237772761474794, + "recall": 0.06589301327398459, + "f1-score": 0.11919482386772107, + "support": 25162.0 + }, + "accuracy": 0.7758937635470683, + "macro avg": { + "precision": 0.7017305060377077, + "recall": 0.5270067829529381, + "f1-score": 0.4954044348090269, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7438059010718815, + "recall": 0.7758937635470683, + "f1-score": 0.6984642441113292, + "support": 109341.0 + }, + "cross-entropy": 0.5001730088224092, + "roc-auc": 0.682790915855999, + "score": 0.682790915855999 + }, + "val": { + "0": { + "precision": 0.6730699855699855, + "recall": 0.9618507539631396, + "f1-score": 0.791956279513981, + "support": 15518.0 + }, + "1": { + "precision": 0.7160671462829736, + "recall": 0.17076518357543177, + "f1-score": 0.27576653121536754, + "support": 8743.0 + }, + "accuracy": 0.6767651786818351, + "macro avg": { + "precision": 0.6945685659264795, + "recall": 0.5663079687692857, + "f1-score": 0.5338614053646742, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6885649847915204, + "recall": 0.6767651786818351, + "f1-score": 0.6059356303496934, + "support": 24261.0 + }, + "cross-entropy": 0.6459983643302272, + "roc-auc": 0.6395464280764918, + "score": 0.6395464280764918 + }, + "test": { + "0": { + "precision": 0.6567702552719201, + "recall": 0.9791395816097388, + "f1-score": 0.7861921708185053, + "support": 16922.0 + }, + "1": { + "precision": 0.7123064384678076, + "recall": 0.09168152732613029, + "f1-score": 0.162453531598513, + "support": 9533.0 + }, + "accuracy": 0.6593460593460594, + "macro avg": { + "precision": 0.6845383468698638, + "recall": 0.5354105544679345, + "f1-score": 0.47432285120850914, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6767825945048211, + "recall": 0.6593460593460594, + "f1-score": 0.5614293491332214, + "support": 26455.0 + }, + "cross-entropy": 0.6822269705952246, + "roc-auc": 0.6014523595237629, + "score": 0.6014523595237629 + } + }, + "time": "0:01:18.936665", + "diversity": { + "train": { + "mean_centered_corr": -0.016698535531759262, + "mean_pairwise_disagreement": 0.024727871167177205, + "member_std": 0.027984458953142166 + }, + "val": { + "mean_centered_corr": 0.0022652181796729565, + "mean_pairwise_disagreement": 0.06734623868654192, + "member_std": 0.05871646851301193 + }, + "test": { + "mean_centered_corr": 0.008645827881991863, + "mean_pairwise_disagreement": 0.044527987269922756, + "member_std": 0.06025338172912598 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.877353" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..b028a7163 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..5c9565e1a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7813747645951036, + "recall": 0.9857803015003742, + "f1-score": 0.8717558134037893, + "support": 84179.0 + }, + "1": { + "precision": 0.6189111747851003, + "recall": 0.0772593593514029, + "f1-score": 0.13737059675652757, + "support": 25162.0 + }, + "accuracy": 0.7767077308603361, + "macro avg": { + "precision": 0.7001429696901019, + "recall": 0.5315198304258886, + "f1-score": 0.5045632050801584, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439879760455265, + "recall": 0.7767077308603361, + "f1-score": 0.7027560711179276, + "support": 109341.0 + }, + "cross-entropy": 0.4956555313079169, + "roc-auc": 0.6916478507195538, + "score": 0.6916478507195538 + }, + "val": { + "0": { + "precision": 0.6748129675810474, + "recall": 0.9590797783219487, + "f1-score": 0.7922178159848827, + "support": 15518.0 + }, + "1": { + "precision": 0.7121486854034451, + "recall": 0.17968660642799955, + "f1-score": 0.28696684628733216, + "support": 8743.0 + }, + "accuracy": 0.6782078232554305, + "macro avg": { + "precision": 0.6934808264922463, + "recall": 0.5693831923749741, + "f1-score": 0.5395923311361075, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6882677378263473, + "recall": 0.6782078232554305, + "f1-score": 0.6101392030643237, + "support": 24261.0 + }, + "cross-entropy": 0.6750260020751795, + "roc-auc": 0.6226529287429354, + "score": 0.6226529287429354 + }, + "test": { + "0": { + "precision": 0.6569412510901451, + "recall": 0.9793168656187212, + "f1-score": 0.7863718325899213, + "support": 16922.0 + }, + "1": { + "precision": 0.7152156224572823, + "recall": 0.09220602118955208, + "f1-score": 0.163352536703215, + "support": 9533.0 + }, + "accuracy": 0.6596484596484596, + "macro avg": { + "precision": 0.6860784367737137, + "recall": 0.5357614434041367, + "f1-score": 0.47486218464656815, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6779402902979667, + "recall": 0.6596484596484596, + "f1-score": 0.561868224663708, + "support": 26455.0 + }, + "cross-entropy": 0.7361401555726955, + "roc-auc": 0.5911485904814773, + "score": 0.5911485904814773 + } + }, + "time": "0:01:18.936665", + "diversity": { + "train": { + "mean_centered_corr": -0.016698535531759262, + "mean_pairwise_disagreement": 0.024727871167177205, + "member_std": 0.027984458953142166 + }, + "val": { + "mean_centered_corr": 0.0022652181796729565, + "mean_pairwise_disagreement": 0.06734623868654192, + "member_std": 0.05871646851301193 + }, + "test": { + "mean_centered_corr": 0.008645827881991863, + "mean_pairwise_disagreement": 0.044527987269922756, + "member_std": 0.06025338172912598 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ae3f4b8c0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 1951424, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.779683735927936, + "recall": 0.9881205526318916, + "f1-score": 0.8716140457503327, + "support": 84179.0 + }, + "1": { + "precision": 0.6237772761474794, + "recall": 0.06589301327398459, + "f1-score": 0.11919482386772107, + "support": 25162.0 + }, + "accuracy": 0.7758937635470683, + "macro avg": { + "precision": 0.7017305060377077, + "recall": 0.5270067829529381, + "f1-score": 0.4954044348090269, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7438059010718815, + "recall": 0.7758937635470683, + "f1-score": 0.6984642441113292, + "support": 109341.0 + }, + "cross-entropy": 0.5001730088224092, + "roc-auc": 0.682790915855999, + "score": 0.682790915855999 + }, + "val": { + "0": { + "precision": 0.6730699855699855, + "recall": 0.9618507539631396, + "f1-score": 0.791956279513981, + "support": 15518.0 + }, + "1": { + "precision": 0.7160671462829736, + "recall": 0.17076518357543177, + "f1-score": 0.27576653121536754, + "support": 8743.0 + }, + "accuracy": 0.6767651786818351, + "macro avg": { + "precision": 0.6945685659264795, + "recall": 0.5663079687692857, + "f1-score": 0.5338614053646742, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6885649847915204, + "recall": 0.6767651786818351, + "f1-score": 0.6059356303496934, + "support": 24261.0 + }, + "cross-entropy": 0.6459983643302272, + "roc-auc": 0.6395464280764918, + "score": 0.6395464280764918 + }, + "test": { + "0": { + "precision": 0.6567702552719201, + "recall": 0.9791395816097388, + "f1-score": 0.7861921708185053, + "support": 16922.0 + }, + "1": { + "precision": 0.7123064384678076, + "recall": 0.09168152732613029, + "f1-score": 0.162453531598513, + "support": 9533.0 + }, + "accuracy": 0.6593460593460594, + "macro avg": { + "precision": 0.6845383468698638, + "recall": 0.5354105544679345, + "f1-score": 0.47432285120850914, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6767825945048211, + "recall": 0.6593460593460594, + "f1-score": 0.5614293491332214, + "support": 26455.0 + }, + "cross-entropy": 0.6822269705952246, + "roc-auc": 0.6014523595237629, + "score": 0.6014523595237629 + } + }, + "time": "0:01:18.936665", + "diversity": { + "train": { + "mean_centered_corr": -0.016698535531759262, + "mean_pairwise_disagreement": 0.024727871167177205, + "member_std": 0.027984458953142166 + }, + "val": { + "mean_centered_corr": 0.0022652181796729565, + "mean_pairwise_disagreement": 0.06734623868654192, + "member_std": 0.05871646851301193 + }, + "test": { + "mean_centered_corr": 0.008645827881991863, + "mean_pairwise_disagreement": 0.044527987269922756, + "member_std": 0.06025338172912598 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.193368" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c46f9f519 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7796513262723779, + "recall": 0.9881443115266277, + "f1-score": 0.8716030366376939, + "support": 84179.0 + }, + "1": { + "precision": 0.6235382874387023, + "recall": 0.06569430092997378, + "f1-score": 0.11886527882644807, + "support": 25162.0 + }, + "accuracy": 0.7758663264466211, + "macro avg": { + "precision": 0.7015948068555401, + "recall": 0.5269193062283007, + "f1-score": 0.49523415773207097, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7437259525961453, + "recall": 0.7758663264466211, + "f1-score": 0.6983799322025179, + "support": 109341.0 + }, + "cross-entropy": 0.500117320045548, + "roc-auc": 0.682915560586896, + "score": 0.682915560586896 + }, + "val": { + "0": { + "precision": 0.6736205183780367, + "recall": 0.9613352236112901, + "f1-score": 0.7921622769753611, + "support": 15518.0 + }, + "1": { + "precision": 0.7163120567375887, + "recall": 0.17328148232872012, + "f1-score": 0.279056916559219, + "support": 8743.0 + }, + "accuracy": 0.6773422365112732, + "macro avg": { + "precision": 0.6949662875578126, + "recall": 0.5673083529700051, + "f1-score": 0.53560959676729, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6890053796730189, + "recall": 0.6773422365112732, + "f1-score": 0.6072531567363632, + "support": 24261.0 + }, + "cross-entropy": 0.6460301333180669, + "roc-auc": 0.6394530055211662, + "score": 0.6394530055211662 + }, + "test": { + "0": { + "precision": 0.656934306569343, + "recall": 0.9786077295827916, + "f1-score": 0.7861381438404937, + "support": 16922.0 + }, + "1": { + "precision": 0.7097032878909383, + "recall": 0.09283541382565824, + "f1-score": 0.16419294990723562, + "support": 9533.0 + }, + "accuracy": 0.6594216594216594, + "macro avg": { + "precision": 0.6833187972301407, + "recall": 0.535721571704225, + "f1-score": 0.4751655468738647, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6759494908044128, + "recall": 0.6594216594216594, + "f1-score": 0.5620215861475908, + "support": 26455.0 + }, + "cross-entropy": 0.6825101151285496, + "roc-auc": 0.6014026252811646, + "score": 0.6014026252811646 + } + }, + "time": "0:01:50.416492", + "diversity": { + "train": { + "mean_centered_corr": -0.01670769229531288, + "mean_pairwise_disagreement": 0.024668442555119807, + "member_std": 0.02791455015540123 + }, + "val": { + "mean_centered_corr": 0.002592463046312332, + "mean_pairwise_disagreement": 0.06743399402465924, + "member_std": 0.05864376202225685 + }, + "test": { + "mean_centered_corr": 0.009284000843763351, + "mean_pairwise_disagreement": 0.0444435468629017, + "member_std": 0.060093436390161514 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.879612" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..f6aef79c4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..0574b3d9b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7814041960149158, + "recall": 0.9857803015003742, + "f1-score": 0.871774130006566, + "support": 84179.0 + }, + "1": { + "precision": 0.61939586645469, + "recall": 0.07741832922661156, + "f1-score": 0.13763380082665067, + "support": 25162.0 + }, + "accuracy": 0.7767443136609323, + "macro avg": { + "precision": 0.7004000312348029, + "recall": 0.5315993153634929, + "f1-score": 0.5047039654166083, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441221738238402, + "recall": 0.7767443136609323, + "f1-score": 0.7028307422304798, + "support": 109341.0 + }, + "cross-entropy": 0.4956439792677601, + "roc-auc": 0.6916789260357137, + "score": 0.6916789260357137 + }, + "val": { + "0": { + "precision": 0.6749659863945578, + "recall": 0.9590797783219487, + "f1-score": 0.7923232538330494, + "support": 15518.0 + }, + "1": { + "precision": 0.7127996381727725, + "recall": 0.18025849250829235, + "f1-score": 0.28774876757348916, + "support": 8743.0 + }, + "accuracy": 0.6784139153373727, + "macro avg": { + "precision": 0.6938828122836651, + "recall": 0.5696691354151205, + "f1-score": 0.5400360107032692, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6886001984013561, + "recall": 0.6784139153373727, + "f1-score": 0.6104884270176941, + "support": 24261.0 + }, + "cross-entropy": 0.6753608510436575, + "roc-auc": 0.6226365549199251, + "score": 0.6226365549199251 + }, + "test": { + "0": { + "precision": 0.6569933396764985, + "recall": 0.9793168656187212, + "f1-score": 0.7864091491481991, + "support": 16922.0 + }, + "1": { + "precision": 0.7156783103168156, + "recall": 0.0924158187349208, + "f1-score": 0.16369379412857674, + "support": 9533.0 + }, + "accuracy": 0.6597240597240597, + "macro avg": { + "precision": 0.6863358249966571, + "recall": 0.535866342176821, + "f1-score": 0.47505147163838796, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6781403374128109, + "recall": 0.6597240597240597, + "f1-score": 0.5620150656327177, + "support": 26455.0 + }, + "cross-entropy": 0.7367195601018909, + "roc-auc": 0.5911320299643263, + "score": 0.5911320299643263 + } + }, + "time": "0:01:50.416492", + "diversity": { + "train": { + "mean_centered_corr": -0.01670769229531288, + "mean_pairwise_disagreement": 0.024668442555119807, + "member_std": 0.02791455015540123 + }, + "val": { + "mean_centered_corr": 0.002592463046312332, + "mean_pairwise_disagreement": 0.06743399402465924, + "member_std": 0.05864376202225685 + }, + "test": { + "mean_centered_corr": 0.009284000843763351, + "mean_pairwise_disagreement": 0.0444435468629017, + "member_std": 0.060093436390161514 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3d6cf2d76 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7796513262723779, + "recall": 0.9881443115266277, + "f1-score": 0.8716030366376939, + "support": 84179.0 + }, + "1": { + "precision": 0.6235382874387023, + "recall": 0.06569430092997378, + "f1-score": 0.11886527882644807, + "support": 25162.0 + }, + "accuracy": 0.7758663264466211, + "macro avg": { + "precision": 0.7015948068555401, + "recall": 0.5269193062283007, + "f1-score": 0.49523415773207097, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7437259525961453, + "recall": 0.7758663264466211, + "f1-score": 0.6983799322025179, + "support": 109341.0 + }, + "cross-entropy": 0.500117320045548, + "roc-auc": 0.682915560586896, + "score": 0.682915560586896 + }, + "val": { + "0": { + "precision": 0.6736205183780367, + "recall": 0.9613352236112901, + "f1-score": 0.7921622769753611, + "support": 15518.0 + }, + "1": { + "precision": 0.7163120567375887, + "recall": 0.17328148232872012, + "f1-score": 0.279056916559219, + "support": 8743.0 + }, + "accuracy": 0.6773422365112732, + "macro avg": { + "precision": 0.6949662875578126, + "recall": 0.5673083529700051, + "f1-score": 0.53560959676729, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6890053796730189, + "recall": 0.6773422365112732, + "f1-score": 0.6072531567363632, + "support": 24261.0 + }, + "cross-entropy": 0.6460301333180669, + "roc-auc": 0.6394530055211662, + "score": 0.6394530055211662 + }, + "test": { + "0": { + "precision": 0.656934306569343, + "recall": 0.9786077295827916, + "f1-score": 0.7861381438404937, + "support": 16922.0 + }, + "1": { + "precision": 0.7097032878909383, + "recall": 0.09283541382565824, + "f1-score": 0.16419294990723562, + "support": 9533.0 + }, + "accuracy": 0.6594216594216594, + "macro avg": { + "precision": 0.6833187972301407, + "recall": 0.535721571704225, + "f1-score": 0.4751655468738647, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6759494908044128, + "recall": 0.6594216594216594, + "f1-score": 0.5620215861475908, + "support": 26455.0 + }, + "cross-entropy": 0.6825101151285496, + "roc-auc": 0.6014026252811646, + "score": 0.6014026252811646 + } + }, + "time": "0:01:50.416492", + "diversity": { + "train": { + "mean_centered_corr": -0.01670769229531288, + "mean_pairwise_disagreement": 0.024668442555119807, + "member_std": 0.02791455015540123 + }, + "val": { + "mean_centered_corr": 0.002592463046312332, + "mean_pairwise_disagreement": 0.06743399402465924, + "member_std": 0.05864376202225685 + }, + "test": { + "mean_centered_corr": 0.009284000843763351, + "mean_pairwise_disagreement": 0.0444435468629017, + "member_std": 0.060093436390161514 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.157879" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a28e18620 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.779705661792276, + "recall": 0.9881205526318916, + "f1-score": 0.8716277461372007, + "support": 84179.0 + }, + "1": { + "precision": 0.6242014280345735, + "recall": 0.06601224068039106, + "f1-score": 0.11939762067354347, + "support": 25162.0 + }, + "accuracy": 0.7759212006475156, + "macro avg": { + "precision": 0.7019535449134247, + "recall": 0.5270663966561413, + "f1-score": 0.4955126834053721, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439203888405806, + "recall": 0.7759212006475156, + "f1-score": 0.6985214601427746, + "support": 109341.0 + }, + "cross-entropy": 0.5001030608040564, + "roc-auc": 0.6829250128727141, + "score": 0.6829250128727141 + }, + "val": { + "0": { + "precision": 0.6734841302090387, + "recall": 0.9612707823173089, + "f1-score": 0.792046088087716, + "support": 15518.0 + }, + "1": { + "precision": 0.7154356060606061, + "recall": 0.17282397346448589, + "f1-score": 0.2783970520497467, + "support": 8743.0 + }, + "accuracy": 0.677136144429331, + "macro avg": { + "precision": 0.6944598681348224, + "recall": 0.5670473778908974, + "f1-score": 0.5352215700687314, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6886022932431368, + "recall": 0.677136144429331, + "f1-score": 0.6069410420434489, + "support": 24261.0 + }, + "cross-entropy": 0.6461835001274551, + "roc-auc": 0.6394467257565004, + "score": 0.6394467257565004 + }, + "test": { + "0": { + "precision": 0.6570771001150748, + "recall": 0.9785486349131308, + "f1-score": 0.7862213042755739, + "support": 16922.0 + }, + "1": { + "precision": 0.7105263157894737, + "recall": 0.0934648064617644, + "f1-score": 0.1651988504681561, + "support": 9533.0 + }, + "accuracy": 0.6596106596106596, + "macro avg": { + "precision": 0.6838017079522742, + "recall": 0.5360067206874476, + "f1-score": 0.475710077371865, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6763374052756889, + "recall": 0.6596106596106596, + "f1-score": 0.5624372539204004, + "support": 26455.0 + }, + "cross-entropy": 0.682694952589047, + "roc-auc": 0.6014832923257777, + "score": 0.6014832923257777 + } + }, + "time": "0:01:49.700702", + "diversity": { + "train": { + "mean_centered_corr": -0.01669180765748024, + "mean_pairwise_disagreement": 0.024674711785060706, + "member_std": 0.027917660772800446 + }, + "val": { + "mean_centered_corr": 0.0025723774451762438, + "mean_pairwise_disagreement": 0.06741305240988126, + "member_std": 0.05856572464108467 + }, + "test": { + "mean_centered_corr": 0.00913835596293211, + "mean_pairwise_disagreement": 0.04448012754464367, + "member_std": 0.060016945004463196 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.877772" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..224095ef4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..17d82eeb4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7814041960149158, + "recall": 0.9857803015003742, + "f1-score": 0.871774130006566, + "support": 84179.0 + }, + "1": { + "precision": 0.61939586645469, + "recall": 0.07741832922661156, + "f1-score": 0.13763380082665067, + "support": 25162.0 + }, + "accuracy": 0.7767443136609323, + "macro avg": { + "precision": 0.7004000312348029, + "recall": 0.5315993153634929, + "f1-score": 0.5047039654166083, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441221738238402, + "recall": 0.7767443136609323, + "f1-score": 0.7028307422304798, + "support": 109341.0 + }, + "cross-entropy": 0.49563994809608797, + "roc-auc": 0.6916786795898221, + "score": 0.6916786795898221 + }, + "val": { + "0": { + "precision": 0.6749671157073525, + "recall": 0.9589508957339863, + "f1-score": 0.792280047916944, + "support": 15518.0 + }, + "1": { + "precision": 0.7122854561878952, + "recall": 0.1803728697243509, + "f1-score": 0.28785251437437254, + "support": 8743.0 + }, + "accuracy": 0.6783726969209843, + "macro avg": { + "precision": 0.6936262859476239, + "recall": 0.5696618827291686, + "f1-score": 0.5400662811456582, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6884156236345355, + "recall": 0.6783726969209843, + "f1-score": 0.6104981788364154, + "support": 24261.0 + }, + "cross-entropy": 0.6753967669711587, + "roc-auc": 0.622598916870318, + "score": 0.622598916870318 + }, + "test": { + "0": { + "precision": 0.6570318385472423, + "recall": 0.9792577709490604, + "f1-score": 0.78641767316043, + "support": 16922.0 + }, + "1": { + "precision": 0.7155591572123177, + "recall": 0.09262561628028952, + "f1-score": 0.16401968979288567, + "support": 9533.0 + }, + "accuracy": 0.6597618597618597, + "macro avg": { + "precision": 0.68629549787978, + "recall": 0.5359416936146749, + "f1-score": 0.4752186814766578, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.678122026747362, + "recall": 0.6597618597618597, + "f1-score": 0.5621379538089728, + "support": 26455.0 + }, + "cross-entropy": 0.7367599370342945, + "roc-auc": 0.5911027119909538, + "score": 0.5911027119909538 + } + }, + "time": "0:01:49.700702", + "diversity": { + "train": { + "mean_centered_corr": -0.01669180765748024, + "mean_pairwise_disagreement": 0.024674711785060706, + "member_std": 0.027917660772800446 + }, + "val": { + "mean_centered_corr": 0.0025723774451762438, + "mean_pairwise_disagreement": 0.06741305240988126, + "member_std": 0.05856572464108467 + }, + "test": { + "mean_centered_corr": 0.00913835596293211, + "mean_pairwise_disagreement": 0.04448012754464367, + "member_std": 0.060016945004463196 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..40942574c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.779705661792276, + "recall": 0.9881205526318916, + "f1-score": 0.8716277461372007, + "support": 84179.0 + }, + "1": { + "precision": 0.6242014280345735, + "recall": 0.06601224068039106, + "f1-score": 0.11939762067354347, + "support": 25162.0 + }, + "accuracy": 0.7759212006475156, + "macro avg": { + "precision": 0.7019535449134247, + "recall": 0.5270663966561413, + "f1-score": 0.4955126834053721, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439203888405806, + "recall": 0.7759212006475156, + "f1-score": 0.6985214601427746, + "support": 109341.0 + }, + "cross-entropy": 0.5001030608040564, + "roc-auc": 0.6829250128727141, + "score": 0.6829250128727141 + }, + "val": { + "0": { + "precision": 0.6734841302090387, + "recall": 0.9612707823173089, + "f1-score": 0.792046088087716, + "support": 15518.0 + }, + "1": { + "precision": 0.7154356060606061, + "recall": 0.17282397346448589, + "f1-score": 0.2783970520497467, + "support": 8743.0 + }, + "accuracy": 0.677136144429331, + "macro avg": { + "precision": 0.6944598681348224, + "recall": 0.5670473778908974, + "f1-score": 0.5352215700687314, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6886022932431368, + "recall": 0.677136144429331, + "f1-score": 0.6069410420434489, + "support": 24261.0 + }, + "cross-entropy": 0.6461835001274551, + "roc-auc": 0.6394467257565004, + "score": 0.6394467257565004 + }, + "test": { + "0": { + "precision": 0.6570771001150748, + "recall": 0.9785486349131308, + "f1-score": 0.7862213042755739, + "support": 16922.0 + }, + "1": { + "precision": 0.7105263157894737, + "recall": 0.0934648064617644, + "f1-score": 0.1651988504681561, + "support": 9533.0 + }, + "accuracy": 0.6596106596106596, + "macro avg": { + "precision": 0.6838017079522742, + "recall": 0.5360067206874476, + "f1-score": 0.475710077371865, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6763374052756889, + "recall": 0.6596106596106596, + "f1-score": 0.5624372539204004, + "support": 26455.0 + }, + "cross-entropy": 0.682694952589047, + "roc-auc": 0.6014832923257777, + "score": 0.6014832923257777 + } + }, + "time": "0:01:49.700702", + "diversity": { + "train": { + "mean_centered_corr": -0.01669180765748024, + "mean_pairwise_disagreement": 0.024674711785060706, + "member_std": 0.027917660772800446 + }, + "val": { + "mean_centered_corr": 0.0025723774451762438, + "mean_pairwise_disagreement": 0.06741305240988126, + "member_std": 0.05856572464108467 + }, + "test": { + "mean_centered_corr": 0.00913835596293211, + "mean_pairwise_disagreement": 0.04448012754464367, + "member_std": 0.060016945004463196 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.147273" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..11f463d90 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7797560449657319, + "recall": 0.9879898787108423, + "f1-score": 0.8716083798824134, + "support": 84179.0 + }, + "1": { + "precision": 0.6230425055928411, + "recall": 0.06640966536841268, + "f1-score": 0.12002585835368482, + "support": 25162.0 + }, + "accuracy": 0.7759120549473665, + "macro avg": { + "precision": 0.7013992752792866, + "recall": 0.5271997720396275, + "f1-score": 0.4958171191180491, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436924816390686, + "recall": 0.7759120549473665, + "f1-score": 0.6986511231653002, + "support": 109341.0 + }, + "cross-entropy": 0.5001252189253765, + "roc-auc": 0.682885703100578, + "score": 0.682885703100578 + }, + "val": { + "0": { + "precision": 0.6736970463372776, + "recall": 0.9612707823173089, + "f1-score": 0.7921933085501859, + "support": 15518.0 + }, + "1": { + "precision": 0.7163756488909863, + "recall": 0.1736246139768958, + "f1-score": 0.2795065365494384, + "support": 8743.0 + }, + "accuracy": 0.6774246733440501, + "macro avg": { + "precision": 0.6950363476141319, + "recall": 0.5674476981471024, + "f1-score": 0.5358499225498121, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6890772459220875, + "recall": 0.6774246733440501, + "f1-score": 0.6074350361128364, + "support": 24261.0 + }, + "cross-entropy": 0.6460050337477455, + "roc-auc": 0.6396100364908871, + "score": 0.6396100364908871 + }, + "test": { + "0": { + "precision": 0.6570385653070941, + "recall": 0.9786077295827916, + "f1-score": 0.7862127902008261, + "support": 16922.0 + }, + "1": { + "precision": 0.7106314948041567, + "recall": 0.09325500891639568, + "f1-score": 0.1648738872403561, + "support": 9533.0 + }, + "accuracy": 0.6595728595728596, + "macro avg": { + "precision": 0.6838350300556254, + "recall": 0.5359313692495936, + "f1-score": 0.4755433387205911, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6763506574218361, + "recall": 0.6595728595728596, + "f1-score": 0.5623147081020864, + "support": 26455.0 + }, + "cross-entropy": 0.6825652834520043, + "roc-auc": 0.6014864444960832, + "score": 0.6014864444960832 + } + }, + "time": "0:01:49.922203", + "diversity": { + "train": { + "mean_centered_corr": -0.016829919070005417, + "mean_pairwise_disagreement": 0.02467144809770912, + "member_std": 0.027938073500990868 + }, + "val": { + "mean_centered_corr": 0.002263085450977087, + "mean_pairwise_disagreement": 0.06737382843299547, + "member_std": 0.05858324468135834 + }, + "test": { + "mean_centered_corr": 0.008585128001868725, + "mean_pairwise_disagreement": 0.044407347229927875, + "member_std": 0.06004700809717178 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.887817" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..4e35430e6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..14a93ea2b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7814009548112506, + "recall": 0.9858040603951104, + "f1-score": 0.871781403313408, + "support": 84179.0 + }, + "1": { + "precision": 0.6196690006365373, + "recall": 0.0773785867578094, + "f1-score": 0.13757772752967778, + "support": 25162.0 + }, + "accuracy": 0.7767534593610814, + "macro avg": { + "precision": 0.700534977723894, + "recall": 0.5315913235764599, + "f1-score": 0.5046795654215429, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441825332590045, + "recall": 0.7767534593610814, + "f1-score": 0.702823437956678, + "support": 109341.0 + }, + "cross-entropy": 0.4956495574180934, + "roc-auc": 0.6916692445363317, + "score": 0.6916692445363317 + }, + "val": { + "0": { + "precision": 0.6749965984851921, + "recall": 0.9590797783219487, + "f1-score": 0.7923443447706764, + "support": 15518.0 + }, + "1": { + "precision": 0.7129294755877035, + "recall": 0.1803728697243509, + "f1-score": 0.2879050661798266, + "support": 8743.0 + }, + "accuracy": 0.6784551337537612, + "macro avg": { + "precision": 0.6939630370364478, + "recall": 0.5697263240231498, + "f1-score": 0.5401247054752515, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6886665685815302, + "recall": 0.6784551337537612, + "f1-score": 0.6105582430963925, + "support": 24261.0 + }, + "cross-entropy": 0.6751780870468839, + "roc-auc": 0.6226645595746754, + "score": 0.6226645595746754 + }, + "test": { + "0": { + "precision": 0.6569797407128415, + "recall": 0.9792577709490604, + "f1-score": 0.7863803535413454, + "support": 16922.0 + }, + "1": { + "precision": 0.7150974025974026, + "recall": 0.0924158187349208, + "f1-score": 0.16367858801672086, + "support": 9533.0 + }, + "accuracy": 0.6596862596862597, + "macro avg": { + "precision": 0.686038571655122, + "recall": 0.5358367948419906, + "f1-score": 0.4750294707790331, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6779223100095916, + "recall": 0.6596862596862597, + "f1-score": 0.5619911669699508, + "support": 26455.0 + }, + "cross-entropy": 0.7364469465377833, + "roc-auc": 0.5911489283247057, + "score": 0.5911489283247057 + } + }, + "time": "0:01:49.922203", + "diversity": { + "train": { + "mean_centered_corr": -0.016829919070005417, + "mean_pairwise_disagreement": 0.02467144809770912, + "member_std": 0.027938073500990868 + }, + "val": { + "mean_centered_corr": 0.002263085450977087, + "mean_pairwise_disagreement": 0.06737382843299547, + "member_std": 0.05858324468135834 + }, + "test": { + "mean_centered_corr": 0.008585128001868725, + "mean_pairwise_disagreement": 0.044407347229927875, + "member_std": 0.06004700809717178 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..13ed1d5d4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2120128, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7797560449657319, + "recall": 0.9879898787108423, + "f1-score": 0.8716083798824134, + "support": 84179.0 + }, + "1": { + "precision": 0.6230425055928411, + "recall": 0.06640966536841268, + "f1-score": 0.12002585835368482, + "support": 25162.0 + }, + "accuracy": 0.7759120549473665, + "macro avg": { + "precision": 0.7013992752792866, + "recall": 0.5271997720396275, + "f1-score": 0.4958171191180491, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436924816390686, + "recall": 0.7759120549473665, + "f1-score": 0.6986511231653002, + "support": 109341.0 + }, + "cross-entropy": 0.5001252189253765, + "roc-auc": 0.682885703100578, + "score": 0.682885703100578 + }, + "val": { + "0": { + "precision": 0.6736970463372776, + "recall": 0.9612707823173089, + "f1-score": 0.7921933085501859, + "support": 15518.0 + }, + "1": { + "precision": 0.7163756488909863, + "recall": 0.1736246139768958, + "f1-score": 0.2795065365494384, + "support": 8743.0 + }, + "accuracy": 0.6774246733440501, + "macro avg": { + "precision": 0.6950363476141319, + "recall": 0.5674476981471024, + "f1-score": 0.5358499225498121, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6890772459220875, + "recall": 0.6774246733440501, + "f1-score": 0.6074350361128364, + "support": 24261.0 + }, + "cross-entropy": 0.6460050337477455, + "roc-auc": 0.6396100364908871, + "score": 0.6396100364908871 + }, + "test": { + "0": { + "precision": 0.6570385653070941, + "recall": 0.9786077295827916, + "f1-score": 0.7862127902008261, + "support": 16922.0 + }, + "1": { + "precision": 0.7106314948041567, + "recall": 0.09325500891639568, + "f1-score": 0.1648738872403561, + "support": 9533.0 + }, + "accuracy": 0.6595728595728596, + "macro avg": { + "precision": 0.6838350300556254, + "recall": 0.5359313692495936, + "f1-score": 0.4755433387205911, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6763506574218361, + "recall": 0.6595728595728596, + "f1-score": 0.5623147081020864, + "support": 26455.0 + }, + "cross-entropy": 0.6825652834520043, + "roc-auc": 0.6014864444960832, + "score": 0.6014864444960832 + } + }, + "time": "0:01:49.922203", + "diversity": { + "train": { + "mean_centered_corr": -0.016829919070005417, + "mean_pairwise_disagreement": 0.02467144809770912, + "member_std": 0.027938073500990868 + }, + "val": { + "mean_centered_corr": 0.002263085450977087, + "mean_pairwise_disagreement": 0.06737382843299547, + "member_std": 0.05858324468135834 + }, + "test": { + "mean_centered_corr": 0.008585128001868725, + "mean_pairwise_disagreement": 0.044407347229927875, + "member_std": 0.06004700809717178 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.225544" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..93e34fb24 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2457536, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7797414237631374, + "recall": 0.9879898787108423, + "f1-score": 0.8715992454412073, + "support": 84179.0 + }, + "1": { + "precision": 0.6227611940298508, + "recall": 0.06633018043080836, + "f1-score": 0.11989081244163494, + "support": 25162.0 + }, + "accuracy": 0.7758937635470683, + "macro avg": { + "precision": 0.701251308896494, + "recall": 0.5271600295708253, + "f1-score": 0.4957450289414211, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436164885554023, + "recall": 0.7758937635470683, + "f1-score": 0.6986130134592862, + "support": 109341.0 + }, + "cross-entropy": 0.5001025680620461, + "roc-auc": 0.6830259884114023, + "score": 0.6830259884114023 + }, + "val": { + "0": { + "precision": 0.6732070365358592, + "recall": 0.9617863126691584, + "f1-score": 0.7920292931437062, + "support": 15518.0 + }, + "1": { + "precision": 0.7164036346245816, + "recall": 0.17133706965572457, + "f1-score": 0.2765368285028614, + "support": 8743.0 + }, + "accuracy": 0.6769300523473888, + "macro avg": { + "precision": 0.6948053355802204, + "recall": 0.5665616911624415, + "f1-score": 0.5342830608232838, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6887739075259132, + "recall": 0.6769300523473888, + "f1-score": 0.6062599259142059, + "support": 24261.0 + }, + "cross-entropy": 0.6465081837078666, + "roc-auc": 0.639398116545268, + "score": 0.639398116545268 + }, + "test": { + "0": { + "precision": 0.6569128900519409, + "recall": 0.979080486940078, + "f1-score": 0.7862753007616924, + "support": 16922.0 + }, + "1": { + "precision": 0.713128038897893, + "recall": 0.09231091996223645, + "f1-score": 0.16346243150366863, + "support": 9533.0 + }, + "accuracy": 0.6595350595350595, + "macro avg": { + "precision": 0.6850204644749169, + "recall": 0.5356957034511572, + "f1-score": 0.47486886613268053, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6771698930361957, + "recall": 0.6595350595350595, + "f1-score": 0.5618460782088011, + "support": 26455.0 + }, + "cross-entropy": 0.6832483314751614, + "roc-auc": 0.6013710911801927, + "score": 0.6013710911801927 + } + }, + "time": "0:02:55.144244", + "diversity": { + "train": { + "mean_centered_corr": -0.016687747091054916, + "mean_pairwise_disagreement": 0.024748319920131486, + "member_std": 0.02787911146879196 + }, + "val": { + "mean_centered_corr": 0.00230225152336061, + "mean_pairwise_disagreement": 0.0672119464266957, + "member_std": 0.058351658284664154 + }, + "test": { + "mean_centered_corr": 0.008990149945020676, + "mean_pairwise_disagreement": 0.0442228434163918, + "member_std": 0.059662897139787674 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.808438" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..3cbee9e39 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..d1e209b0f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2457536, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7814086204785806, + "recall": 0.9857209042635336, + "f1-score": 0.8717536560766516, + "support": 84179.0 + }, + "1": { + "precision": 0.6186548223350253, + "recall": 0.07749781416421588, + "f1-score": 0.13774104683195593, + "support": 25162.0 + }, + "accuracy": 0.776716876560485, + "macro avg": { + "precision": 0.700031721406803, + "recall": 0.5316093592138748, + "f1-score": 0.5047473514543037, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.743955047995357, + "recall": 0.776716876560485, + "f1-score": 0.7028396597366232, + "support": 109341.0 + }, + "cross-entropy": 0.4956245931886986, + "roc-auc": 0.6917220323493016, + "score": 0.6917220323493016 + }, + "val": { + "0": { + "precision": 0.6747823721436343, + "recall": 0.9590797783219487, + "f1-score": 0.7921967317826156, + "support": 15518.0 + }, + "1": { + "precision": 0.7120181405895691, + "recall": 0.179572229211941, + "f1-score": 0.28681037632444284, + "support": 8743.0 + }, + "accuracy": 0.6781666048390421, + "macro avg": { + "precision": 0.6934002563666017, + "recall": 0.5693260037669449, + "f1-score": 0.5395035540535292, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6882011233708224, + "recall": 0.6781666048390421, + "f1-score": 0.6100693295415371, + "support": 24261.0 + }, + "cross-entropy": 0.6759411491858018, + "roc-auc": 0.6225693459597093, + "score": 0.6225693459597093 + }, + "test": { + "0": { + "precision": 0.6568891707626446, + "recall": 0.9793168656187212, + "f1-score": 0.7863345195729537, + "support": 16922.0 + }, + "1": { + "precision": 0.7147514262428688, + "recall": 0.09199622364418336, + "f1-score": 0.16301115241635689, + "support": 9533.0 + }, + "accuracy": 0.6595728595728596, + "macro avg": { + "precision": 0.6858202985027567, + "recall": 0.5356565446314523, + "f1-score": 0.4746728359946553, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6777397049336135, + "recall": 0.6595728595728596, + "f1-score": 0.5617213402456493, + "support": 26455.0 + }, + "cross-entropy": 0.7376821008229536, + "roc-auc": 0.5910522493707531, + "score": 0.5910522493707531 + } + }, + "time": "0:02:55.144244", + "diversity": { + "train": { + "mean_centered_corr": -0.016687747091054916, + "mean_pairwise_disagreement": 0.024748319920131486, + "member_std": 0.02787911146879196 + }, + "val": { + "mean_centered_corr": 0.00230225152336061, + "mean_pairwise_disagreement": 0.0672119464266957, + "member_std": 0.058351658284664154 + }, + "test": { + "mean_centered_corr": 0.008990149945020676, + "mean_pairwise_disagreement": 0.0442228434163918, + "member_std": 0.059662897139787674 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..cdcaf1088 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2457536, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7797414237631374, + "recall": 0.9879898787108423, + "f1-score": 0.8715992454412073, + "support": 84179.0 + }, + "1": { + "precision": 0.6227611940298508, + "recall": 0.06633018043080836, + "f1-score": 0.11989081244163494, + "support": 25162.0 + }, + "accuracy": 0.7758937635470683, + "macro avg": { + "precision": 0.701251308896494, + "recall": 0.5271600295708253, + "f1-score": 0.4957450289414211, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436164885554023, + "recall": 0.7758937635470683, + "f1-score": 0.6986130134592862, + "support": 109341.0 + }, + "cross-entropy": 0.5001025680620461, + "roc-auc": 0.6830259884114023, + "score": 0.6830259884114023 + }, + "val": { + "0": { + "precision": 0.6732070365358592, + "recall": 0.9617863126691584, + "f1-score": 0.7920292931437062, + "support": 15518.0 + }, + "1": { + "precision": 0.7164036346245816, + "recall": 0.17133706965572457, + "f1-score": 0.2765368285028614, + "support": 8743.0 + }, + "accuracy": 0.6769300523473888, + "macro avg": { + "precision": 0.6948053355802204, + "recall": 0.5665616911624415, + "f1-score": 0.5342830608232838, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6887739075259132, + "recall": 0.6769300523473888, + "f1-score": 0.6062599259142059, + "support": 24261.0 + }, + "cross-entropy": 0.6465081837078666, + "roc-auc": 0.639398116545268, + "score": 0.639398116545268 + }, + "test": { + "0": { + "precision": 0.6569128900519409, + "recall": 0.979080486940078, + "f1-score": 0.7862753007616924, + "support": 16922.0 + }, + "1": { + "precision": 0.713128038897893, + "recall": 0.09231091996223645, + "f1-score": 0.16346243150366863, + "support": 9533.0 + }, + "accuracy": 0.6595350595350595, + "macro avg": { + "precision": 0.6850204644749169, + "recall": 0.5356957034511572, + "f1-score": 0.47486886613268053, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6771698930361957, + "recall": 0.6595350595350595, + "f1-score": 0.5618460782088011, + "support": 26455.0 + }, + "cross-entropy": 0.6832483314751614, + "roc-auc": 0.6013710911801927, + "score": 0.6013710911801927 + } + }, + "time": "0:02:55.144244", + "diversity": { + "train": { + "mean_centered_corr": -0.016687747091054916, + "mean_pairwise_disagreement": 0.024748319920131486, + "member_std": 0.02787911146879196 + }, + "val": { + "mean_centered_corr": 0.00230225152336061, + "mean_pairwise_disagreement": 0.0672119464266957, + "member_std": 0.058351658284664154 + }, + "test": { + "mean_centered_corr": 0.008990149945020676, + "mean_pairwise_disagreement": 0.0442228434163918, + "member_std": 0.059662897139787674 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.157803" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..29cd98b02 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2457536, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7796848489393414, + "recall": 0.9880849142897873, + "f1-score": 0.8716008760439699, + "support": 84179.0 + }, + "1": { + "precision": 0.6232156273478587, + "recall": 0.06593275574278674, + "f1-score": 0.11924956871765383, + "support": 25162.0 + }, + "accuracy": 0.7758754721467702, + "macro avg": { + "precision": 0.7014502381436001, + "recall": 0.527008835016287, + "f1-score": 0.4954252223808118, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436775090239859, + "recall": 0.7758754721467702, + "f1-score": 0.6984667031815965, + "support": 109341.0 + }, + "cross-entropy": 0.5001057107203039, + "roc-auc": 0.6829994192308994, + "score": 0.6829994192308994 + }, + "val": { + "0": { + "precision": 0.6728264298913779, + "recall": 0.961979636551102, + "f1-score": 0.7918313221058215, + "support": 15518.0 + }, + "1": { + "precision": 0.7155255544840887, + "recall": 0.16973578863090472, + "f1-score": 0.27438291578071555, + "support": 8743.0 + }, + "accuracy": 0.676476649767116, + "macro avg": { + "precision": 0.6941759921877333, + "recall": 0.5658577125910034, + "f1-score": 0.5331071189432686, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6882140250570375, + "recall": 0.676476649767116, + "f1-score": 0.6053570870577856, + "support": 24261.0 + }, + "cross-entropy": 0.6469691623792566, + "roc-auc": 0.6392187010153481, + "score": 0.6392187010153481 + }, + "test": { + "0": { + "precision": 0.6566040725774502, + "recall": 0.9794350549580428, + "f1-score": 0.7861682952281567, + "support": 16922.0 + }, + "1": { + "precision": 0.7131079967023908, + "recall": 0.09073743837197105, + "f1-score": 0.16099013586450772, + "support": 9533.0 + }, + "accuracy": 0.6591948591948592, + "macro avg": { + "precision": 0.6848560346399205, + "recall": 0.5350862466650069, + "f1-score": 0.4735792155463322, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6769651350867323, + "recall": 0.6591948591948592, + "f1-score": 0.5608867456831306, + "support": 26455.0 + }, + "cross-entropy": 0.6838126131386375, + "roc-auc": 0.6012738729168664, + "score": 0.6012738729168664 + } + }, + "time": "0:02:54.252595", + "diversity": { + "train": { + "mean_centered_corr": -0.01680082082748413, + "mean_pairwise_disagreement": 0.024706850807963605, + "member_std": 0.027859199792146683 + }, + "val": { + "mean_centered_corr": 0.002759142080321908, + "mean_pairwise_disagreement": 0.06733643269232048, + "member_std": 0.0583425834774971 + }, + "test": { + "mean_centered_corr": 0.009294640272855759, + "mean_pairwise_disagreement": 0.04442342748794362, + "member_std": 0.05963234603404999 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.876008" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..6054af7db --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..5cd64046b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2457536, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7813927209378972, + "recall": 0.9857565426056379, + "f1-score": 0.8717576980049797, + "support": 84179.0 + }, + "1": { + "precision": 0.6188811188811189, + "recall": 0.0773785867578094, + "f1-score": 0.13755828740991946, + "support": 25162.0 + }, + "accuracy": 0.776716876560485, + "macro avg": { + "precision": 0.700136919909508, + "recall": 0.5315675646817237, + "f1-score": 0.5046579927074496, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7439948836128988, + "recall": 0.776716876560485, + "f1-score": 0.7028007141709842, + "support": 109341.0 + }, + "cross-entropy": 0.4956186570036448, + "roc-auc": 0.69174046857932, + "score": 0.69174046857932 + }, + "val": { + "0": { + "precision": 0.6748435657930534, + "recall": 0.9590797783219487, + "f1-score": 0.7922389013094858, + "support": 15518.0 + }, + "1": { + "precision": 0.712279111916629, + "recall": 0.1798009836440581, + "f1-score": 0.28712328767123285, + "support": 8743.0 + }, + "accuracy": 0.678249041671819, + "macro avg": { + "precision": 0.6935613388548412, + "recall": 0.5694403809830034, + "f1-score": 0.5396810944903594, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6883343114242483, + "recall": 0.678249041671819, + "f1-score": 0.610209067005902, + "support": 24261.0 + }, + "cross-entropy": 0.6759987255155949, + "roc-auc": 0.6225384372823319, + "score": 0.6225384372823319 + }, + "test": { + "0": { + "precision": 0.6569276511397423, + "recall": 0.9792577709490604, + "f1-score": 0.7863430374641137, + "support": 16922.0 + }, + "1": { + "precision": 0.7146341463414634, + "recall": 0.09220602118955208, + "f1-score": 0.1633373594722661, + "support": 9533.0 + }, + "accuracy": 0.6596106596106596, + "macro avg": { + "precision": 0.6857808987406029, + "recall": 0.5357318960693063, + "f1-score": 0.4748401984681899, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6777220574432014, + "recall": 0.6596106596106596, + "f1-score": 0.5618443367158135, + "support": 26455.0 + }, + "cross-entropy": 0.7377377305054865, + "roc-auc": 0.591035974005685, + "score": 0.591035974005685 + } + }, + "time": "0:02:54.252595", + "diversity": { + "train": { + "mean_centered_corr": -0.01680082082748413, + "mean_pairwise_disagreement": 0.024706850807963605, + "member_std": 0.027859199792146683 + }, + "val": { + "mean_centered_corr": 0.002759142080321908, + "mean_pairwise_disagreement": 0.06733643269232048, + "member_std": 0.0583425834774971 + }, + "test": { + "mean_centered_corr": 0.009294640272855759, + "mean_pairwise_disagreement": 0.04442342748794362, + "member_std": 0.05963234603404999 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7e28a06fe --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2457536, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7796848489393414, + "recall": 0.9880849142897873, + "f1-score": 0.8716008760439699, + "support": 84179.0 + }, + "1": { + "precision": 0.6232156273478587, + "recall": 0.06593275574278674, + "f1-score": 0.11924956871765383, + "support": 25162.0 + }, + "accuracy": 0.7758754721467702, + "macro avg": { + "precision": 0.7014502381436001, + "recall": 0.527008835016287, + "f1-score": 0.4954252223808118, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436775090239859, + "recall": 0.7758754721467702, + "f1-score": 0.6984667031815965, + "support": 109341.0 + }, + "cross-entropy": 0.5001057107203039, + "roc-auc": 0.6829994192308994, + "score": 0.6829994192308994 + }, + "val": { + "0": { + "precision": 0.6728264298913779, + "recall": 0.961979636551102, + "f1-score": 0.7918313221058215, + "support": 15518.0 + }, + "1": { + "precision": 0.7155255544840887, + "recall": 0.16973578863090472, + "f1-score": 0.27438291578071555, + "support": 8743.0 + }, + "accuracy": 0.676476649767116, + "macro avg": { + "precision": 0.6941759921877333, + "recall": 0.5658577125910034, + "f1-score": 0.5331071189432686, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6882140250570375, + "recall": 0.676476649767116, + "f1-score": 0.6053570870577856, + "support": 24261.0 + }, + "cross-entropy": 0.6469691623792566, + "roc-auc": 0.6392187010153481, + "score": 0.6392187010153481 + }, + "test": { + "0": { + "precision": 0.6566040725774502, + "recall": 0.9794350549580428, + "f1-score": 0.7861682952281567, + "support": 16922.0 + }, + "1": { + "precision": 0.7131079967023908, + "recall": 0.09073743837197105, + "f1-score": 0.16099013586450772, + "support": 9533.0 + }, + "accuracy": 0.6591948591948592, + "macro avg": { + "precision": 0.6848560346399205, + "recall": 0.5350862466650069, + "f1-score": 0.4735792155463322, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6769651350867323, + "recall": 0.6591948591948592, + "f1-score": 0.5608867456831306, + "support": 26455.0 + }, + "cross-entropy": 0.6838126131386375, + "roc-auc": 0.6012738729168664, + "score": 0.6012738729168664 + } + }, + "time": "0:02:54.252595", + "diversity": { + "train": { + "mean_centered_corr": -0.01680082082748413, + "mean_pairwise_disagreement": 0.024706850807963605, + "member_std": 0.027859199792146683 + }, + "val": { + "mean_centered_corr": 0.002759142080321908, + "mean_pairwise_disagreement": 0.06733643269232048, + "member_std": 0.0583425834774971 + }, + "test": { + "mean_centered_corr": 0.009294640272855759, + "mean_pairwise_disagreement": 0.04442342748794362, + "member_std": 0.05963234603404999 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.186604" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4cf780715 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2457536, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7796294039908898, + "recall": 0.9881443115266277, + "f1-score": 0.8715893373569722, + "support": 84179.0 + }, + "1": { + "precision": 0.6231117824773413, + "recall": 0.06557507352356728, + "f1-score": 0.1186623516720604, + "support": 25162.0 + }, + "accuracy": 0.7758388893461738, + "macro avg": { + "precision": 0.7013705932341155, + "recall": 0.5268596925250975, + "f1-score": 0.4951258445145163, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436109260866827, + "recall": 0.7758388893461738, + "f1-score": 0.6983226870263208, + "support": 109341.0 + }, + "cross-entropy": 0.5001170885588134, + "roc-auc": 0.6829371288514838, + "score": 0.6829371288514838 + }, + "val": { + "0": { + "precision": 0.673176672229489, + "recall": 0.9617863126691584, + "f1-score": 0.7920082782774815, + "support": 15518.0 + }, + "1": { + "precision": 0.716267942583732, + "recall": 0.171222692439666, + "f1-score": 0.2763777346995292, + "support": 8743.0 + }, + "accuracy": 0.6768888339310004, + "macro avg": { + "precision": 0.6947223074066105, + "recall": 0.5665045025544122, + "f1-score": 0.5341930064885054, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.688705585988491, + "recall": 0.6768888339310004, + "f1-score": 0.6061891511804106, + "support": 24261.0 + }, + "cross-entropy": 0.6463251283606344, + "roc-auc": 0.6393903110631306, + "score": 0.6393903110631306 + }, + "test": { + "0": { + "precision": 0.6568744053282588, + "recall": 0.9791395816097388, + "f1-score": 0.78626678688369, + "support": 16922.0 + }, + "1": { + "precision": 0.7132412672623883, + "recall": 0.09210112241686773, + "f1-score": 0.16313638052768487, + "support": 9533.0 + }, + "accuracy": 0.6594972594972595, + "macro avg": { + "precision": 0.6850578362953236, + "recall": 0.5356203520133033, + "f1-score": 0.47470158370568744, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6771860777840538, + "recall": 0.6594972594972595, + "f1-score": 0.5617231405487138, + "support": 26455.0 + }, + "cross-entropy": 0.6828939339210119, + "roc-auc": 0.6013715932958167, + "score": 0.6013715932958167 + } + }, + "time": "0:02:54.360546", + "diversity": { + "train": { + "mean_centered_corr": -0.01661285012960434, + "mean_pairwise_disagreement": 0.024667170270220036, + "member_std": 0.027912361547350883 + }, + "val": { + "mean_centered_corr": 0.0025234981440007687, + "mean_pairwise_disagreement": 0.06728490967183492, + "member_std": 0.058532536029815674 + }, + "test": { + "mean_centered_corr": 0.009481742046773434, + "mean_pairwise_disagreement": 0.04442228434163918, + "member_std": 0.05997689068317413 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:04.892264" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..7d7b01333 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/ecom-offers" +num_policy = "noisy-quantile" + +[optimizer] +lr = 0.00024262819114537424 +weight_decay = 0.0001501852317298042 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 1 +d_block = 960 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 16 + +[bins] +n_bins = 47 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..1992163e5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2457536, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7814062544139665, + "recall": 0.9857921809477423, + "f1-score": 0.8717800563096189, + "support": 84179.0 + }, + "1": { + "precision": 0.6195928753180662, + "recall": 0.07741832922661156, + "f1-score": 0.13763866318095103, + "support": 25162.0 + }, + "accuracy": 0.7767534593610814, + "macro avg": { + "precision": 0.7004995648660164, + "recall": 0.5316052550871769, + "f1-score": 0.504709359745285, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7441690950244325, + "recall": 0.7767534593610814, + "f1-score": 0.702836423693276, + "support": 109341.0 + }, + "cross-entropy": 0.4956384516885499, + "roc-auc": 0.6916909499513633, + "score": 0.6916909499513633 + }, + "val": { + "0": { + "precision": 0.6749659863945578, + "recall": 0.9590797783219487, + "f1-score": 0.7923232538330494, + "support": 15518.0 + }, + "1": { + "precision": 0.7127996381727725, + "recall": 0.18025849250829235, + "f1-score": 0.28774876757348916, + "support": 8743.0 + }, + "accuracy": 0.6784139153373727, + "macro avg": { + "precision": 0.6938828122836651, + "recall": 0.5696691354151205, + "f1-score": 0.5400360107032692, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.6886001984013561, + "recall": 0.6784139153373727, + "f1-score": 0.6104884270176941, + "support": 24261.0 + }, + "cross-entropy": 0.6754694410618766, + "roc-auc": 0.6226251931156621, + "score": 0.6226251931156621 + }, + "test": { + "0": { + "precision": 0.6569276511397423, + "recall": 0.9792577709490604, + "f1-score": 0.7863430374641137, + "support": 16922.0 + }, + "1": { + "precision": 0.7146341463414634, + "recall": 0.09220602118955208, + "f1-score": 0.1633373594722661, + "support": 9533.0 + }, + "accuracy": 0.6596106596106596, + "macro avg": { + "precision": 0.6857808987406029, + "recall": 0.5357318960693063, + "f1-score": 0.4748401984681899, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6777220574432014, + "recall": 0.6596106596106596, + "f1-score": 0.5618443367158135, + "support": 26455.0 + }, + "cross-entropy": 0.736898658236868, + "roc-auc": 0.5910938288836818, + "score": 0.5910938288836818 + } + }, + "time": "0:02:54.360546", + "diversity": { + "train": { + "mean_centered_corr": -0.01661285012960434, + "mean_pairwise_disagreement": 0.024667170270220036, + "member_std": 0.027912361547350883 + }, + "val": { + "mean_centered_corr": 0.0025234981440007687, + "mean_pairwise_disagreement": 0.06728490967183492, + "member_std": 0.058532536029815674 + }, + "test": { + "mean_centered_corr": 0.009481742046773434, + "mean_pairwise_disagreement": 0.04442228434163918, + "member_std": 0.05997689068317413 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3dc14a84b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/ecom-offers/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,228 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/ecom-offers", + "num_policy": "noisy-quantile" + }, + "optimizer": { + "lr": 0.00024262819114537424, + "weight_decay": 0.0001501852317298042 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 1, + "d_block": 960, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 16 + } + }, + "bins": { + "n_bins": 47 + } + }, + "n_parameters": 2457536, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 107, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 1676, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 960, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 960 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 214, + "metrics": { + "train": { + "0": { + "precision": 0.7796294039908898, + "recall": 0.9881443115266277, + "f1-score": 0.8715893373569722, + "support": 84179.0 + }, + "1": { + "precision": 0.6231117824773413, + "recall": 0.06557507352356728, + "f1-score": 0.1186623516720604, + "support": 25162.0 + }, + "accuracy": 0.7758388893461738, + "macro avg": { + "precision": 0.7013705932341155, + "recall": 0.5268596925250975, + "f1-score": 0.4951258445145163, + "support": 109341.0 + }, + "weighted avg": { + "precision": 0.7436109260866827, + "recall": 0.7758388893461738, + "f1-score": 0.6983226870263208, + "support": 109341.0 + }, + "cross-entropy": 0.5001170885588134, + "roc-auc": 0.6829371288514838, + "score": 0.6829371288514838 + }, + "val": { + "0": { + "precision": 0.673176672229489, + "recall": 0.9617863126691584, + "f1-score": 0.7920082782774815, + "support": 15518.0 + }, + "1": { + "precision": 0.716267942583732, + "recall": 0.171222692439666, + "f1-score": 0.2763777346995292, + "support": 8743.0 + }, + "accuracy": 0.6768888339310004, + "macro avg": { + "precision": 0.6947223074066105, + "recall": 0.5665045025544122, + "f1-score": 0.5341930064885054, + "support": 24261.0 + }, + "weighted avg": { + "precision": 0.688705585988491, + "recall": 0.6768888339310004, + "f1-score": 0.6061891511804106, + "support": 24261.0 + }, + "cross-entropy": 0.6463251283606344, + "roc-auc": 0.6393903110631306, + "score": 0.6393903110631306 + }, + "test": { + "0": { + "precision": 0.6568744053282588, + "recall": 0.9791395816097388, + "f1-score": 0.78626678688369, + "support": 16922.0 + }, + "1": { + "precision": 0.7132412672623883, + "recall": 0.09210112241686773, + "f1-score": 0.16313638052768487, + "support": 9533.0 + }, + "accuracy": 0.6594972594972595, + "macro avg": { + "precision": 0.6850578362953236, + "recall": 0.5356203520133033, + "f1-score": 0.47470158370568744, + "support": 26455.0 + }, + "weighted avg": { + "precision": 0.6771860777840538, + "recall": 0.6594972594972595, + "f1-score": 0.5617231405487138, + "support": 26455.0 + }, + "cross-entropy": 0.6828939339210119, + "roc-auc": 0.6013715932958167, + "score": 0.6013715932958167 + } + }, + "time": "0:02:54.360546", + "diversity": { + "train": { + "mean_centered_corr": -0.01661285012960434, + "mean_pairwise_disagreement": 0.024667170270220036, + "member_std": 0.027912361547350883 + }, + "val": { + "mean_centered_corr": 0.0025234981440007687, + "mean_pairwise_disagreement": 0.06728490967183492, + "member_std": 0.058532536029815674 + }, + "test": { + "mean_centered_corr": 0.009481742046773434, + "mean_pairwise_disagreement": 0.04442228434163918, + "member_std": 0.05997689068317413 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 11 + ], + "head_selection_time": "0:00:11.225895" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5d789d8e2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-best-head-evaluation/0/report.json @@ -0,0 +1,680 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9513552811475944, + "recall": 0.9636150815239791, + "f1-score": 0.9574459372022485, + "support": 181971.0 + }, + "1": { + "precision": 0.8344915508449156, + "recall": 0.7882830763418263, + "f1-score": 0.8107294209075565, + "support": 42349.0 + }, + "accuracy": 0.9305144436519258, + "macro avg": { + "precision": 0.8929234159962549, + "recall": 0.8759490789329027, + "f1-score": 0.8840876790549025, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9292927717209353, + "recall": 0.9305144436519258, + "f1-score": 0.9297475699208473, + "support": 224320.0 + }, + "cross-entropy": 0.15873224362284377, + "roc-auc": 0.9722680689598284, + "score": 0.9722680689598284 + }, + "val": { + "0": { + "precision": 0.9436578686709782, + "recall": 0.9674451944328296, + "f1-score": 0.9554034922133081, + "support": 16741.0 + }, + "1": { + "precision": 0.8168067226890756, + "recall": 0.7153370621136297, + "f1-score": 0.7627118644067796, + "support": 3397.0 + }, + "accuracy": 0.9249180653490913, + "macro avg": { + "precision": 0.8802322956800269, + "recall": 0.8413911282732296, + "f1-score": 0.8590576783100439, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9222598478694823, + "recall": 0.9249180653490913, + "f1-score": 0.9228990995398164, + "support": 20138.0 + }, + "cross-entropy": 0.17990073731774503, + "roc-auc": 0.9569685349939213, + "score": 0.9569685349939213 + }, + "test": { + "0": { + "precision": 0.9437623162043353, + "recall": 0.9471362287974443, + "f1-score": 0.9454462624805436, + "support": 13147.0 + }, + "1": { + "precision": 0.7758787487907127, + "recall": 0.764294790343075, + "f1-score": 0.7700432069131061, + "support": 3148.0 + }, + "accuracy": 0.9118134397054312, + "macro avg": { + "precision": 0.859820532497524, + "recall": 0.8557155095702597, + "f1-score": 0.8577447346968249, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9113292097165732, + "recall": 0.9118134397054312, + "f1-score": 0.9115604804046742, + "support": 16295.0 + }, + "cross-entropy": 0.19410702240157, + "roc-auc": 0.9582203301945192, + "score": 0.9582203301945192 + } + }, + "time": "0:14:29.029077", + "diversity": { + "train": { + "mean_centered_corr": 0.09211483597755432, + "mean_pairwise_disagreement": 0.03578823167364594, + "member_std": 0.03491802513599396 + }, + "val": { + "mean_centered_corr": 0.1013437807559967, + "mean_pairwise_disagreement": 0.03309443068632885, + "member_std": 0.03697067126631737 + }, + "test": { + "mean_centered_corr": 0.12410959601402283, + "mean_pairwise_disagreement": 0.0401679220817785, + "member_std": 0.03913382440805435 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6 + ], + "head_selection_time": "0:00:01.134116" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..5b9249ca9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0.toml @@ -0,0 +1,75 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..0f201487a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/0/report.json @@ -0,0 +1,676 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9439755183585884, + "recall": 0.9814915563468904, + "f1-score": 0.9623680537968715, + "support": 181971.0 + }, + "1": { + "precision": 0.9040920351966284, + "recall": 0.7496989303171268, + "f1-score": 0.8196886375958485, + "support": 42349.0 + }, + "accuracy": 0.9377318116975749, + "macro avg": { + "precision": 0.9240337767776083, + "recall": 0.8655952433320087, + "f1-score": 0.89102834569636, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9364459818552635, + "recall": 0.9377318116975749, + "f1-score": 0.9354318439328508, + "support": 224320.0 + }, + "cross-entropy": 0.14691183931212085, + "roc-auc": 0.9764623137919726, + "score": 0.9764623137919726 + }, + "val": { + "0": { + "precision": 0.934692250754427, + "recall": 0.980586583836091, + "f1-score": 0.957089552238806, + "support": 16741.0 + }, + "1": { + "precision": 0.8737864077669902, + "recall": 0.6623491315866942, + "f1-score": 0.753516409912927, + "support": 3397.0 + }, + "accuracy": 0.9269043599165756, + "macro avg": { + "precision": 0.9042393292607086, + "recall": 0.8214678577113925, + "f1-score": 0.8553029810758666, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.924418283695716, + "recall": 0.9269043599165756, + "f1-score": 0.9227495996873605, + "support": 20138.0 + }, + "cross-entropy": 0.17471226841771598, + "roc-auc": 0.958474253988237, + "score": 0.958474253988237 + }, + "test": { + "0": { + "precision": 0.9308116741687237, + "recall": 0.9752034684718948, + "f1-score": 0.9524906207050258, + "support": 13147.0 + }, + "1": { + "precision": 0.8706862356207854, + "recall": 0.6972681067344345, + "f1-score": 0.7743870171106015, + "support": 3148.0 + }, + "accuracy": 0.9215096655415772, + "macro avg": { + "precision": 0.9007489548947545, + "recall": 0.8362357876031646, + "f1-score": 0.8634388189078137, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9191961552642187, + "recall": 0.9215096655415772, + "f1-score": 0.9180831249016966, + "support": 16295.0 + }, + "cross-entropy": 0.17874492250186327, + "roc-auc": 0.962487069534998, + "score": 0.962487069534998 + } + }, + "time": "0:14:29.029077", + "diversity": { + "train": { + "mean_centered_corr": 0.09211483597755432, + "mean_pairwise_disagreement": 0.03578823167364594, + "member_std": 0.03491802513599396 + }, + "val": { + "mean_centered_corr": 0.1013437807559967, + "mean_pairwise_disagreement": 0.03309443068632885, + "member_std": 0.03697067126631737 + }, + "test": { + "mean_centered_corr": 0.12410959601402283, + "mean_pairwise_disagreement": 0.0401679220817785, + "member_std": 0.03913382440805435 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..6d2ffd9c7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9442336152219873, + "recall": 0.9817443438789697, + "f1-score": 0.9626236963555881, + "support": 181971.0 + }, + "1": { + "precision": 0.9054100227790433, + "recall": 0.7508559824316985, + "f1-score": 0.8209219171539583, + "support": 42349.0 + }, + "accuracy": 0.9381553138373752, + "macro avg": { + "precision": 0.9248218190005153, + "recall": 0.8663001631553341, + "f1-score": 0.8917728067547732, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9369041737260607, + "recall": 0.9381553138373752, + "f1-score": 0.9358720529559367, + "support": 224320.0 + }, + "cross-entropy": 0.14721044455331747, + "roc-auc": 0.9763538308497102, + "score": 0.9763538308497102 + }, + "val": { + "0": { + "precision": 0.9343600818367811, + "recall": 0.9820799235410071, + "f1-score": 0.9576258846142645, + "support": 16741.0 + }, + "1": { + "precision": 0.8819826907946499, + "recall": 0.6599941124521637, + "f1-score": 0.7550092608183195, + "support": 3397.0 + }, + "accuracy": 0.9277485351077565, + "macro avg": { + "precision": 0.9081713863157155, + "recall": 0.8210370179965853, + "f1-score": 0.8563175727162919, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9255247457870185, + "recall": 0.9277485351077565, + "f1-score": 0.9234472834108269, + "support": 20138.0 + }, + "cross-entropy": 0.17362982513958533, + "roc-auc": 0.9592737644154757, + "score": 0.9592737644154757 + }, + "test": { + "0": { + "precision": 0.9306966618287373, + "recall": 0.975507720392485, + "f1-score": 0.9525754818583578, + "support": 13147.0 + }, + "1": { + "precision": 0.8719681908548708, + "recall": 0.6966327827191868, + "f1-score": 0.7745011478015186, + "support": 3148.0 + }, + "accuracy": 0.9216324025774778, + "macro avg": { + "precision": 0.9013324263418041, + "recall": 0.8360702515558359, + "f1-score": 0.8635383148299383, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.919351020427956, + "recall": 0.9216324025774778, + "f1-score": 0.9181736405812219, + "support": 16295.0 + }, + "cross-entropy": 0.17835468537288718, + "roc-auc": 0.9627526496640615, + "score": 0.9627526496640615 + } + }, + "time": "0:14:29.029077", + "diversity": { + "train": { + "mean_centered_corr": 0.09211483597755432, + "mean_pairwise_disagreement": 0.03578823167364594, + "member_std": 0.03491802513599396 + }, + "val": { + "mean_centered_corr": 0.1013437807559967, + "mean_pairwise_disagreement": 0.03309443068632885, + "member_std": 0.03697067126631737 + }, + "test": { + "mean_centered_corr": 0.12410959601402283, + "mean_pairwise_disagreement": 0.0401679220817785, + "member_std": 0.03913382440805435 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6, + 31, + 30, + 28, + 8, + 20, + 12, + 30, + 8, + 21, + 15 + ], + "head_selection_time": "0:00:14.268703" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f2506d24e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,321 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9506353233351531, + "recall": 0.9760401382637893, + "f1-score": 0.9631702390151977, + "support": 181971.0 + }, + "1": { + "precision": 0.8836899109000693, + "recall": 0.7822144560674396, + "f1-score": 0.8298615895284023, + "support": 42349.0 + }, + "accuracy": 0.9394481098430814, + "macro avg": { + "precision": 0.9171626171176113, + "recall": 0.8791272971656144, + "f1-score": 0.8965159142718, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9379968101788881, + "recall": 0.9394481098430814, + "f1-score": 0.9380031206257705, + "support": 224320.0 + }, + "cross-entropy": 0.1407259896841775, + "roc-auc": 0.9773294079585453, + "score": 0.9773294079585453 + }, + "val": { + "0": { + "precision": 0.9386192203604538, + "recall": 0.9737172211934771, + "f1-score": 0.955846135803917, + "support": 16741.0 + }, + "1": { + "precision": 0.8412125586430892, + "recall": 0.6861937003238151, + "f1-score": 0.7558365758754864, + "support": 3397.0 + }, + "accuracy": 0.9252160095342139, + "macro avg": { + "precision": 0.8899158895017715, + "recall": 0.8299554607586461, + "f1-score": 0.8558413558397017, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9221880737791702, + "recall": 0.9252160095342139, + "f1-score": 0.9221073099484756, + "support": 20138.0 + }, + "cross-entropy": 0.18155767784095098, + "roc-auc": 0.9560637742304589, + "score": 0.9560637742304589 + }, + "test": { + "0": { + "precision": 0.9381688466111772, + "recall": 0.960219061382825, + "f1-score": 0.949065894823892, + "support": 13147.0 + }, + "1": { + "precision": 0.8157802042972878, + "recall": 0.735705209656925, + "f1-score": 0.7736762986470687, + "support": 3148.0 + }, + "accuracy": 0.916845658177355, + "macro avg": { + "precision": 0.8769745254542325, + "recall": 0.847962135519875, + "f1-score": 0.8613710967354804, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9145248180131947, + "recall": 0.916845658177355, + "f1-score": 0.9151827129420486, + "support": 16295.0 + }, + "cross-entropy": 0.1860567271079682, + "roc-auc": 0.9602624689888717, + "score": 0.9602624689888717 + } + }, + "time": "0:13:27.087006", + "diversity": { + "train": { + "mean_centered_corr": 0.08523335307836533, + "mean_pairwise_disagreement": 0.038875824714693294, + "member_std": 0.03867573291063309 + }, + "val": { + "mean_centered_corr": 0.09791728109121323, + "mean_pairwise_disagreement": 0.036375120539246936, + "member_std": 0.04205026477575302 + }, + "test": { + "mean_centered_corr": 0.11845514178276062, + "mean_pairwise_disagreement": 0.044347043918082925, + "member_std": 0.04443347081542015 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 31 + ], + "head_selection_time": "0:00:01.210122" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..e9e4aca66 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0.toml @@ -0,0 +1,64 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..80d109a53 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/0/report.json @@ -0,0 +1,317 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.94720362658795, + "recall": 0.9874870171620753, + "f1-score": 0.9669259391790271, + "support": 181971.0 + }, + "1": { + "precision": 0.9342097659635943, + "recall": 0.7634891024581454, + "f1-score": 0.8402655959666836, + "support": 42349.0 + }, + "accuracy": 0.945198823109843, + "macro avg": { + "precision": 0.9407066962757722, + "recall": 0.8754880598101104, + "f1-score": 0.9035957675728554, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.944750537235325, + "recall": 0.945198823109843, + "f1-score": 0.9430139434822568, + "support": 224320.0 + }, + "cross-entropy": 0.13349501270533368, + "roc-auc": 0.9819246121431815, + "score": 0.9819246121431815 + }, + "val": { + "0": { + "precision": 0.9319724044333861, + "recall": 0.9844692670688728, + "f1-score": 0.9575018155410312, + "support": 16741.0 + }, + "1": { + "precision": 0.8940505297473512, + "recall": 0.6458639976449808, + "f1-score": 0.7499572722611519, + "support": 3397.0 + }, + "accuracy": 0.9273512761942596, + "macro avg": { + "precision": 0.9130114670903686, + "recall": 0.8151666323569269, + "f1-score": 0.8537295439010916, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9255755125718079, + "recall": 0.9273512761942596, + "f1-score": 0.9224919429855764, + "support": 20138.0 + }, + "cross-entropy": 0.1755486957932297, + "roc-auc": 0.9580700807398709, + "score": 0.9580700807398709 + }, + "test": { + "0": { + "precision": 0.9290154691340177, + "recall": 0.9775614208564691, + "f1-score": 0.9526703976872614, + "support": 13147.0 + }, + "1": { + "precision": 0.880130028443722, + "recall": 0.6880559085133418, + "f1-score": 0.7723301836334463, + "support": 3148.0 + }, + "accuracy": 0.9216324025774778, + "macro avg": { + "precision": 0.9045727487888698, + "recall": 0.8328086646849054, + "f1-score": 0.8625002906603538, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9195713839978991, + "recall": 0.9216324025774778, + "f1-score": 0.917830815371127, + "support": 16295.0 + }, + "cross-entropy": 0.17967216581259585, + "roc-auc": 0.9619997518046595, + "score": 0.9619997518046595 + } + }, + "time": "0:13:27.087006", + "diversity": { + "train": { + "mean_centered_corr": 0.08523335307836533, + "mean_pairwise_disagreement": 0.038875824714693294, + "member_std": 0.03867573291063309 + }, + "val": { + "mean_centered_corr": 0.09791728109121323, + "mean_pairwise_disagreement": 0.036375120539246936, + "member_std": 0.04205026477575302 + }, + "test": { + "mean_centered_corr": 0.11845514178276062, + "mean_pairwise_disagreement": 0.044347043918082925, + "member_std": 0.04443347081542015 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7386df965 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,333 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9475126453288841, + "recall": 0.9861846118337537, + "f1-score": 0.9664619287016417, + "support": 181971.0 + }, + "1": { + "precision": 0.9280109959337953, + "recall": 0.765260100592694, + "f1-score": 0.8388140440786324, + "support": 42349.0 + }, + "accuracy": 0.9444766405135521, + "macro avg": { + "precision": 0.9377618206313397, + "recall": 0.8757223562132239, + "f1-score": 0.902637986390137, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9438309613496018, + "recall": 0.9444766405135521, + "f1-score": 0.9423634967031582, + "support": 224320.0 + }, + "cross-entropy": 0.1342408923499761, + "roc-auc": 0.9812870495057586, + "score": 0.9812870495057586 + }, + "val": { + "0": { + "precision": 0.9332577903682719, + "recall": 0.9839316647751031, + "f1-score": 0.9579250385275218, + "support": 16741.0 + }, + "1": { + "precision": 0.8918810289389068, + "recall": 0.6532234324403886, + "f1-score": 0.7541206457094307, + "support": 3397.0 + }, + "accuracy": 0.9281457940212533, + "macro avg": { + "precision": 0.9125694096535893, + "recall": 0.8185775486077458, + "f1-score": 0.8560228421184762, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9262781072529898, + "recall": 0.9281457940212533, + "f1-score": 0.9235460772402513, + "support": 20138.0 + }, + "cross-entropy": 0.17438799592759188, + "roc-auc": 0.9587104803714672, + "score": 0.9587104803714672 + }, + "test": { + "0": { + "precision": 0.9292336665218021, + "recall": 0.9758119723130753, + "f1-score": 0.9519534003635959, + "support": 13147.0 + }, + "1": { + "precision": 0.8722378465247087, + "recall": 0.6896442185514613, + "f1-score": 0.7702678729820827, + "support": 3148.0 + }, + "accuracy": 0.9205277692543725, + "macro avg": { + "precision": 0.9007357565232554, + "recall": 0.8327280954322682, + "f1-score": 0.8611106366728393, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9182227526616702, + "recall": 0.9205277692543725, + "f1-score": 0.916853919529168, + "support": 16295.0 + }, + "cross-entropy": 0.17983663015934195, + "roc-auc": 0.9619912225060596, + "score": 0.9619912225060596 + } + }, + "time": "0:13:27.087006", + "diversity": { + "train": { + "mean_centered_corr": 0.08523335307836533, + "mean_pairwise_disagreement": 0.038875824714693294, + "member_std": 0.03867573291063309 + }, + "val": { + "mean_centered_corr": 0.09791728109121323, + "mean_pairwise_disagreement": 0.036375120539246936, + "member_std": 0.04205026477575302 + }, + "test": { + "mean_centered_corr": 0.11845514178276062, + "mean_pairwise_disagreement": 0.044347043918082925, + "member_std": 0.04443347081542015 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 31, + 2, + 4, + 20, + 28, + 22, + 7, + 16, + 25, + 9, + 29, + 1, + 2 + ], + "head_selection_time": "0:00:13.607257" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-best-head-evaluation/0/report.json new file mode 100644 index 000000000..105ac355c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-best-head-evaluation/0/report.json @@ -0,0 +1,670 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9368912902937575, + "recall": 0.9795626775695029, + "f1-score": 0.9577519275716627, + "support": 181971.0 + }, + "1": { + "precision": 0.8908135404127888, + "recall": 0.7164750053129944, + "f1-score": 0.7941892422457794, + "support": 42349.0 + }, + "accuracy": 0.9298947931526391, + "macro avg": { + "precision": 0.9138524153532732, + "recall": 0.8480188414412486, + "f1-score": 0.875970584908721, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9281923484708743, + "recall": 0.9298947931526391, + "f1-score": 0.926873200035706, + "support": 224320.0 + }, + "cross-entropy": 0.1611006274052141, + "roc-auc": 0.9697728815477382, + "score": 0.9697728815477382 + }, + "val": { + "0": { + "precision": 0.9335682218559982, + "recall": 0.9813033868944507, + "f1-score": 0.9568408177529268, + "support": 16741.0 + }, + "1": { + "precision": 0.8768201495474223, + "recall": 0.6558728289667354, + "f1-score": 0.7504210171775009, + "support": 3397.0 + }, + "accuracy": 0.9264077862747045, + "macro avg": { + "precision": 0.9051941857017103, + "recall": 0.818588107930593, + "f1-score": 0.8536309174652139, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9239956127770316, + "recall": 0.9264077862747045, + "f1-score": 0.92202067361971, + "support": 20138.0 + }, + "cross-entropy": 0.17653579040347495, + "roc-auc": 0.9578664730808395, + "score": 0.9578664730808395 + }, + "test": { + "0": { + "precision": 0.9297203303869005, + "recall": 0.9760401612535179, + "f1-score": 0.952317340161045, + "support": 13147.0 + }, + "1": { + "precision": 0.8736462093862816, + "recall": 0.6918678526048284, + "f1-score": 0.7722035100159547, + "support": 3148.0 + }, + "accuracy": 0.9211414544338754, + "macro avg": { + "precision": 0.9016832698865911, + "recall": 0.8339540069291731, + "f1-score": 0.8622604250884998, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.918887477799607, + "recall": 0.9211414544338754, + "f1-score": 0.9175214925208642, + "support": 16295.0 + }, + "cross-entropy": 0.17954252670408766, + "roc-auc": 0.9618357621457454, + "score": 0.9618357621457454 + } + }, + "time": "0:07:00.265011", + "diversity": { + "train": { + "mean_centered_corr": 0.08075066655874252, + "mean_pairwise_disagreement": 0.03102939600973264, + "member_std": 0.0314626470208168 + }, + "val": { + "mean_centered_corr": 0.08396831154823303, + "mean_pairwise_disagreement": 0.02662335530004261, + "member_std": 0.032109007239341736 + }, + "test": { + "mean_centered_corr": 0.10440877079963684, + "mean_pairwise_disagreement": 0.03210229241108988, + "member_std": 0.03372453153133392 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 31 + ], + "head_selection_time": "0:00:01.603798" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0.toml new file mode 100644 index 000000000..294585115 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0/report.json new file mode 100644 index 000000000..38c78f5bf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-evaluation/0/report.json @@ -0,0 +1,666 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9351769160464812, + "recall": 0.9840194316676834, + "f1-score": 0.9589766659704482, + "support": 181971.0 + }, + "1": { + "precision": 0.9114629319531131, + "recall": 0.7069116153864318, + "f1-score": 0.7962603399207383, + "support": 42349.0 + }, + "accuracy": 0.9317047075606276, + "macro avg": { + "precision": 0.9233199239997971, + "recall": 0.8454655235270576, + "f1-score": 0.8776185029455932, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9306999924000384, + "recall": 0.9317047075606276, + "f1-score": 0.928257721195666, + "support": 224320.0 + }, + "cross-entropy": 0.15826633985081387, + "roc-auc": 0.9715808416321321, + "score": 0.9715808416321321 + }, + "val": { + "0": { + "precision": 0.9273451526032316, + "recall": 0.9873364793023117, + "f1-score": 0.9564009836539853, + "support": 16741.0 + }, + "1": { + "precision": 0.9083837510803803, + "recall": 0.6187812775978805, + "f1-score": 0.7361232708807565, + "support": 3397.0 + }, + "accuracy": 0.9251663521700269, + "macro avg": { + "precision": 0.917864451841806, + "recall": 0.803058878450096, + "f1-score": 0.8462621272673709, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9241466283717724, + "recall": 0.9251663521700269, + "f1-score": 0.9192432028271575, + "support": 20138.0 + }, + "cross-entropy": 0.1764182894477564, + "roc-auc": 0.95846243739381, + "score": 0.95846243739381 + }, + "test": { + "0": { + "precision": 0.9234446347031964, + "recall": 0.9844831520498973, + "f1-score": 0.9529875197879468, + "support": 13147.0 + }, + "1": { + "precision": 0.9104870557261957, + "recall": 0.659148665819568, + "f1-score": 0.7646950433020084, + "support": 3148.0 + }, + "accuracy": 0.9216324025774778, + "macro avg": { + "precision": 0.916965845214696, + "recall": 0.8218159089347327, + "f1-score": 0.8588412815449776, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.920941384711199, + "recall": 0.9216324025774778, + "f1-score": 0.9166116550455268, + "support": 16295.0 + }, + "cross-entropy": 0.18012155300228325, + "roc-auc": 0.9621719566520266, + "score": 0.9621719566520266 + } + }, + "time": "0:07:00.265011", + "diversity": { + "train": { + "mean_centered_corr": 0.08075066655874252, + "mean_pairwise_disagreement": 0.03102939600973264, + "member_std": 0.0314626470208168 + }, + "val": { + "mean_centered_corr": 0.08396831154823303, + "mean_pairwise_disagreement": 0.02662335530004261, + "member_std": 0.032109007239341736 + }, + "test": { + "mean_centered_corr": 0.10440877079963684, + "mean_pairwise_disagreement": 0.03210229241108988, + "member_std": 0.03372453153133392 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f60bcb208 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_esam-greedy-heads-evaluation/0/report.json @@ -0,0 +1,678 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9327077372080825, + "recall": 0.9869979282413132, + "f1-score": 0.9590851564877955, + "support": 181971.0 + }, + "1": { + "precision": 0.9254967408760273, + "recall": 0.6940187489669177, + "f1-score": 0.7932151242814347, + "support": 42349.0 + }, + "accuracy": 0.9316868758915835, + "macro avg": { + "precision": 0.9291022390420549, + "recall": 0.8405083386041154, + "f1-score": 0.876150140384615, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9313463851945921, + "recall": 0.9316868758915835, + "f1-score": 0.9277708287688797, + "support": 224320.0 + }, + "cross-entropy": 0.15549230413668702, + "roc-auc": 0.9720873725388706, + "score": 0.9720873725388706 + }, + "val": { + "0": { + "precision": 0.9275695691202872, + "recall": 0.9875754136550983, + "f1-score": 0.9566324316505135, + "support": 16741.0 + }, + "1": { + "precision": 0.9101123595505618, + "recall": 0.6199587871651457, + "f1-score": 0.7375240763438977, + "support": 3397.0 + }, + "accuracy": 0.9255636110835237, + "macro avg": { + "precision": 0.9188409643354245, + "recall": 0.803767100410122, + "f1-score": 0.8470782539972056, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9246247811121256, + "recall": 0.9255636110835237, + "f1-score": 0.9196719051346443, + "support": 20138.0 + }, + "cross-entropy": 0.1741207203692563, + "roc-auc": 0.959329480009883, + "score": 0.959329480009883 + }, + "test": { + "0": { + "precision": 0.9233678201926507, + "recall": 0.9843310260896022, + "f1-score": 0.9528753405492968, + "support": 13147.0 + }, + "1": { + "precision": 0.9096491228070176, + "recall": 0.658831003811944, + "f1-score": 0.7641857037582903, + "support": 3148.0 + }, + "accuracy": 0.9214482970236268, + "macro avg": { + "precision": 0.9165084714998342, + "recall": 0.8215810149507732, + "f1-score": 0.8585305221537936, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9207175311855951, + "recall": 0.9214482970236268, + "f1-score": 0.9164227491643266, + "support": 16295.0 + }, + "cross-entropy": 0.17746238406307344, + "roc-auc": 0.9630054358452254, + "score": 0.9630054358452254 + } + }, + "time": "0:07:00.265011", + "diversity": { + "train": { + "mean_centered_corr": 0.08075066655874252, + "mean_pairwise_disagreement": 0.03102939600973264, + "member_std": 0.0314626470208168 + }, + "val": { + "mean_centered_corr": 0.08396831154823303, + "mean_pairwise_disagreement": 0.02662335530004261, + "member_std": 0.032109007239341736 + }, + "test": { + "mean_centered_corr": 0.10440877079963684, + "mean_pairwise_disagreement": 0.03210229241108988, + "member_std": 0.03372453153133392 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 31, + 15, + 14, + 28, + 10, + 16, + 15, + 28, + 14 + ], + "head_selection_time": "0:00:19.762715" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..45764d1df --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,644 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9369183420516376, + "recall": 0.9803375263091372, + "f1-score": 0.9581362873447465, + "support": 181971.0 + }, + "1": { + "precision": 0.8945040688760467, + "recall": 0.7163805520791519, + "f1-score": 0.7955943093162001, + "support": 42349.0 + }, + "accuracy": 0.9305055278174037, + "macro avg": { + "precision": 0.9157112054638421, + "recall": 0.8483590391941445, + "f1-score": 0.8768652983304733, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9289110219075885, + "recall": 0.9305055278174037, + "f1-score": 0.9274502574431286, + "support": 224320.0 + }, + "cross-entropy": 0.1606612384407506, + "roc-auc": 0.9700666591028972, + "score": 0.9700666591028972 + }, + "val": { + "0": { + "precision": 0.9329441106450516, + "recall": 0.9831551281285467, + "f1-score": 0.9573917342872932, + "support": 16741.0 + }, + "1": { + "precision": 0.8870192307692307, + "recall": 0.651751545481307, + "f1-score": 0.7513999660614288, + "support": 3397.0 + }, + "accuracy": 0.9272519614658854, + "macro avg": { + "precision": 0.9099816707071411, + "recall": 0.8174533368049268, + "f1-score": 0.8543958501743609, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9251972233206815, + "recall": 0.9272519614658854, + "f1-score": 0.9226437932473059, + "support": 20138.0 + }, + "cross-entropy": 0.1752607407742304, + "roc-auc": 0.9578946605821286, + "score": 0.9578946605821286 + }, + "test": { + "0": { + "precision": 0.9286075218364254, + "recall": 0.9784741766182399, + "f1-score": 0.9528888888888889, + "support": 13147.0 + }, + "1": { + "precision": 0.8841113841113841, + "recall": 0.6858322744599746, + "f1-score": 0.7724508050089446, + "support": 3148.0 + }, + "accuracy": 0.9219392451672292, + "macro avg": { + "precision": 0.9063594529739047, + "recall": 0.8321532255391073, + "f1-score": 0.8626698469489167, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9200113977763805, + "recall": 0.9219392451672292, + "f1-score": 0.9180303992875348, + "support": 16295.0 + }, + "cross-entropy": 0.18002408953514465, + "roc-auc": 0.9613487585255535, + "score": 0.9613487585255535 + } + }, + "time": "0:12:18.387954", + "diversity": { + "train": { + "mean_centered_corr": 0.018650218844413757, + "mean_pairwise_disagreement": 0.022819575146104644, + "member_std": 0.0206756591796875 + }, + "val": { + "mean_centered_corr": 0.01982957310974598, + "mean_pairwise_disagreement": 0.019789761132059755, + "member_std": 0.02188469097018242 + }, + "test": { + "mean_centered_corr": 0.02981637977063656, + "mean_pairwise_disagreement": 0.024840887269991785, + "member_std": 0.022842135280370712 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:01.553339" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0.toml new file mode 100644 index 000000000..6fe9e642b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0.toml @@ -0,0 +1,62 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0/report.json new file mode 100644 index 000000000..75867746e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/0/report.json @@ -0,0 +1,640 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9361022029934762, + "recall": 0.983316022882767, + "f1-score": 0.9591284305317325, + "support": 181971.0 + }, + "1": { + "precision": 0.9084742696934069, + "recall": 0.7115870504616402, + "f1-score": 0.7980667372881356, + "support": 42349.0 + }, + "accuracy": 0.9320167617689016, + "macro avg": { + "precision": 0.9222882363434416, + "recall": 0.8474515366722035, + "f1-score": 0.878597583909934, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9308863713809378, + "recall": 0.9320167617689016, + "f1-score": 0.9287218611345628, + "support": 224320.0 + }, + "cross-entropy": 0.1556527280386954, + "roc-auc": 0.9717201052099171, + "score": 0.9717201052099171 + }, + "val": { + "0": { + "precision": 0.9289128845396771, + "recall": 0.986619676243952, + "f1-score": 0.9568970511557847, + "support": 16741.0 + }, + "1": { + "precision": 0.9049639372083157, + "recall": 0.627906976744186, + "f1-score": 0.7413972888425443, + "support": 3397.0 + }, + "accuracy": 0.9261098420895819, + "macro avg": { + "precision": 0.9169384108739964, + "recall": 0.807263326494069, + "f1-score": 0.8491471699991645, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9248730308260693, + "recall": 0.9261098420895819, + "f1-score": 0.9205452439962815, + "support": 20138.0 + }, + "cross-entropy": 0.17462322365346752, + "roc-auc": 0.9585091762449806, + "score": 0.9585091762449806 + }, + "test": { + "0": { + "precision": 0.924518852400372, + "recall": 0.9828858294667985, + "f1-score": 0.9528093201592686, + "support": 13147.0 + }, + "1": { + "precision": 0.9029335634167386, + "recall": 0.6648665819567979, + "f1-score": 0.765825100622027, + "support": 3148.0 + }, + "accuracy": 0.9214482970236268, + "macro avg": { + "precision": 0.9137262079085553, + "recall": 0.8238762057117982, + "f1-score": 0.8593172103906478, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9203488315522298, + "recall": 0.9214482970236268, + "f1-score": 0.9166861950838935, + "support": 16295.0 + }, + "cross-entropy": 0.17844908711477925, + "roc-auc": 0.9622794548091665, + "score": 0.9622794548091665 + } + }, + "time": "0:12:18.387954", + "diversity": { + "train": { + "mean_centered_corr": 0.018650218844413757, + "mean_pairwise_disagreement": 0.022819575146104644, + "member_std": 0.0206756591796875 + }, + "val": { + "mean_centered_corr": 0.01982957310974598, + "mean_pairwise_disagreement": 0.019789761132059755, + "member_std": 0.02188469097018242 + }, + "test": { + "mean_centered_corr": 0.02981637977063656, + "mean_pairwise_disagreement": 0.024840887269991785, + "member_std": 0.022842135280370712 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..04e352252 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,651 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9362959009617096, + "recall": 0.9833599859318243, + "f1-score": 0.9592510098126721, + "support": 181971.0 + }, + "1": { + "precision": 0.9088006746581532, + "recall": 0.7125079694916054, + "f1-score": 0.7987716906460537, + "support": 42349.0 + }, + "accuracy": 0.9322262838801711, + "macro avg": { + "precision": 0.9225482878099314, + "recall": 0.8479339777117149, + "f1-score": 0.8790113502293628, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9311051228824955, + "recall": 0.9322262838801711, + "f1-score": 0.928954385849641, + "support": 224320.0 + }, + "cross-entropy": 0.15555906132680442, + "roc-auc": 0.9717261720722767, + "score": 0.9717261720722767 + }, + "val": { + "0": { + "precision": 0.9296773830302348, + "recall": 0.9863210083029688, + "f1-score": 0.957161903657759, + "support": 16741.0 + }, + "1": { + "precision": 0.9036600757257046, + "recall": 0.6323226376214307, + "f1-score": 0.744024939383443, + "support": 3397.0 + }, + "accuracy": 0.926606415731453, + "macro avg": { + "precision": 0.9166687293779697, + "recall": 0.8093218229621997, + "f1-score": 0.850593421520601, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9252886258093842, + "recall": 0.926606415731453, + "f1-score": 0.9212086676045337, + "support": 20138.0 + }, + "cross-entropy": 0.17401332164145797, + "roc-auc": 0.9590512800985321, + "score": 0.9590512800985321 + }, + "test": { + "0": { + "precision": 0.9262719172175913, + "recall": 0.9804518141020765, + "f1-score": 0.9525920999150131, + "support": 13147.0 + }, + "1": { + "precision": 0.8919714165615805, + "recall": 0.6740787801778907, + "f1-score": 0.7678668355346481, + "support": 3148.0 + }, + "accuracy": 0.921264191469776, + "macro avg": { + "precision": 0.9091216668895858, + "recall": 0.8272652971399836, + "f1-score": 0.8602294677248306, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9196454688552026, + "recall": 0.921264191469776, + "f1-score": 0.916905378081973, + "support": 16295.0 + }, + "cross-entropy": 0.1777322749820877, + "roc-auc": 0.9625182147641627, + "score": 0.9625182147641627 + } + }, + "time": "0:12:18.387954", + "diversity": { + "train": { + "mean_centered_corr": 0.018650218844413757, + "mean_pairwise_disagreement": 0.022819575146104644, + "member_std": 0.0206756591796875 + }, + "val": { + "mean_centered_corr": 0.01982957310974598, + "mean_pairwise_disagreement": 0.019789761132059755, + "member_std": 0.02188469097018242 + }, + "test": { + "mean_centered_corr": 0.02981637977063656, + "mean_pairwise_disagreement": 0.024840887269991785, + "member_std": 0.022842135280370712 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 15, + 5, + 21, + 14, + 3, + 15, + 2 + ], + "head_selection_time": "0:00:13.539201" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6bd6b2bf1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-best-head-evaluation/0/report.json @@ -0,0 +1,667 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.941693557360632, + "recall": 0.976655620950591, + "f1-score": 0.9588559959859508, + "support": 181971.0 + }, + "1": { + "precision": 0.880650689742365, + "recall": 0.7401591536990247, + "f1-score": 0.8043160298683637, + "support": 42349.0 + }, + "accuracy": 0.9320078459343795, + "macro avg": { + "precision": 0.9111721235514985, + "recall": 0.8584073873248079, + "f1-score": 0.8815860129271573, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9301693758308264, + "recall": 0.9320078459343795, + "f1-score": 0.9296806526143669, + "support": 224320.0 + }, + "cross-entropy": 0.1563500457958513, + "roc-auc": 0.9712464870931182, + "score": 0.9712464870931182 + }, + "val": { + "0": { + "precision": 0.9341423118555233, + "recall": 0.9794516456603548, + "f1-score": 0.9562605703621625, + "support": 16741.0 + }, + "1": { + "precision": 0.8669245647969053, + "recall": 0.6596997350603474, + "f1-score": 0.7492477432296891, + "support": 3397.0 + }, + "accuracy": 0.9255139537193365, + "macro avg": { + "precision": 0.9005334383262142, + "recall": 0.819575690360351, + "f1-score": 0.8527541567959258, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9228036145291688, + "recall": 0.9255139537193365, + "f1-score": 0.9213403909119185, + "support": 20138.0 + }, + "cross-entropy": 0.1776733868160412, + "roc-auc": 0.9573359044742286, + "score": 0.9573359044742286 + }, + "test": { + "0": { + "precision": 0.9269699372792156, + "recall": 0.9780177987373545, + "f1-score": 0.9518099045081057, + "support": 13147.0 + }, + "1": { + "precision": 0.8807755775577558, + "recall": 0.6782083862770013, + "f1-score": 0.7663316582914573, + "support": 3148.0 + }, + "accuracy": 0.9200981896287205, + "macro avg": { + "precision": 0.9038727574184857, + "recall": 0.8281130925071779, + "f1-score": 0.8590707813997815, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.918045736947632, + "recall": 0.9200981896287205, + "f1-score": 0.9159777155489152, + "support": 16295.0 + }, + "cross-entropy": 0.18171269740302973, + "roc-auc": 0.9616503767533748, + "score": 0.9616503767533748 + } + }, + "time": "0:06:50.247903", + "diversity": { + "train": { + "mean_centered_corr": 0.08324653655290604, + "mean_pairwise_disagreement": 0.03557734342644149, + "member_std": 0.03504224866628647 + }, + "val": { + "mean_centered_corr": 0.09014386683702469, + "mean_pairwise_disagreement": 0.03127462845078635, + "member_std": 0.035533007234334946 + }, + "test": { + "mean_centered_corr": 0.11173245310783386, + "mean_pairwise_disagreement": 0.03804637777271872, + "member_std": 0.03772429749369621 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:01.240582" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0.toml new file mode 100644 index 000000000..98da74ee8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0.toml @@ -0,0 +1,68 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0/report.json new file mode 100644 index 000000000..18547af5e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/0/report.json @@ -0,0 +1,663 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9396844355041488, + "recall": 0.9857724582488419, + "f1-score": 0.9621768619910048, + "support": 181971.0 + }, + "1": { + "precision": 0.9225406893250359, + "recall": 0.728116366384094, + "f1-score": 0.8138782943792644, + "support": 42349.0 + }, + "accuracy": 0.9371299928673323, + "macro avg": { + "precision": 0.9311125624145924, + "recall": 0.856944412316468, + "f1-score": 0.8880275781851346, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.936447896154384, + "recall": 0.9371299928673323, + "f1-score": 0.9341798218706874, + "support": 224320.0 + }, + "cross-entropy": 0.1473269491192984, + "roc-auc": 0.9761188653178614, + "score": 0.9761188653178614 + }, + "val": { + "0": { + "precision": 0.9304999718166957, + "recall": 0.9860820739501822, + "f1-score": 0.9574850646714228, + "support": 16741.0 + }, + "1": { + "precision": 0.9027951606174385, + "recall": 0.6370326758904916, + "f1-score": 0.7469796341042457, + "support": 3397.0 + }, + "accuracy": 0.9272023041016982, + "macro avg": { + "precision": 0.9166475662170671, + "recall": 0.8115573749203369, + "f1-score": 0.8522323493878343, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9258265562022416, + "recall": 0.9272023041016982, + "f1-score": 0.921975731687179, + "support": 20138.0 + }, + "cross-entropy": 0.17509016585325612, + "roc-auc": 0.9586943908120914, + "score": 0.9586943908120914 + }, + "test": { + "0": { + "precision": 0.9261995266441942, + "recall": 0.982277325625618, + "f1-score": 0.9534145441122185, + "support": 13147.0 + }, + "1": { + "precision": 0.9009353741496599, + "recall": 0.673125794155019, + "f1-score": 0.7705454545454545, + "support": 3148.0 + }, + "accuracy": 0.9225529303467321, + "macro avg": { + "precision": 0.9135674503969271, + "recall": 0.8277015598903186, + "f1-score": 0.8619799993288365, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9213187931644278, + "recall": 0.9225529303467321, + "f1-score": 0.9180864131544908, + "support": 16295.0 + }, + "cross-entropy": 0.1796124403053473, + "roc-auc": 0.9623596978704976, + "score": 0.9623596978704976 + } + }, + "time": "0:06:50.247903", + "diversity": { + "train": { + "mean_centered_corr": 0.08324653655290604, + "mean_pairwise_disagreement": 0.03557734342644149, + "member_std": 0.03504224866628647 + }, + "val": { + "mean_centered_corr": 0.09014386683702469, + "mean_pairwise_disagreement": 0.03127462845078635, + "member_std": 0.035533007234334946 + }, + "test": { + "mean_centered_corr": 0.11173245310783386, + "mean_pairwise_disagreement": 0.03804637777271872, + "member_std": 0.03772429749369621 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..af02eb05d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-greedy-heads-evaluation/0/report.json @@ -0,0 +1,675 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9394039613420057, + "recall": 0.9833874628374851, + "f1-score": 0.9608926548211629, + "support": 181971.0 + }, + "1": { + "precision": 0.9106388010287032, + "recall": 0.7274315804387352, + "f1-score": 0.8087899393525689, + "support": 42349.0 + }, + "accuracy": 0.9350659771754636, + "macro avg": { + "precision": 0.9250213811853545, + "recall": 0.8554095216381101, + "f1-score": 0.884841297086866, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9339734345316096, + "recall": 0.9350659771754636, + "f1-score": 0.9321774359491074, + "support": 224320.0 + }, + "cross-entropy": 0.1522778155562675, + "roc-auc": 0.9742048378504812, + "score": 0.9742048378504812 + }, + "val": { + "0": { + "precision": 0.931451157538114, + "recall": 0.9853652708918225, + "f1-score": 0.95764999564599, + "support": 16741.0 + }, + "1": { + "precision": 0.8990939044481054, + "recall": 0.6426258463350015, + "f1-score": 0.7495278969957082, + "support": 3397.0 + }, + "accuracy": 0.927549905651008, + "macro avg": { + "precision": 0.9152725309931098, + "recall": 0.813995558613412, + "f1-score": 0.853588946320849, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9259929398031473, + "recall": 0.927549905651008, + "f1-score": 0.9225426975471218, + "support": 20138.0 + }, + "cross-entropy": 0.17448991850377593, + "roc-auc": 0.9594920636885601, + "score": 0.9594920636885601 + }, + "test": { + "0": { + "precision": 0.9245997712978845, + "recall": 0.984026774169012, + "f1-score": 0.9533881130476436, + "support": 13147.0 + }, + "1": { + "precision": 0.9088145896656535, + "recall": 0.6648665819567979, + "f1-score": 0.7679324894514767, + "support": 3148.0 + }, + "accuracy": 0.9223688247928813, + "macro avg": { + "precision": 0.916707180481769, + "recall": 0.824446678062905, + "f1-score": 0.8606603012495602, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9215502621368988, + "recall": 0.9223688247928813, + "f1-score": 0.9175602945093967, + "support": 16295.0 + }, + "cross-entropy": 0.1794172260622648, + "roc-auc": 0.9630834801355295, + "score": 0.9630834801355295 + } + }, + "time": "0:06:50.247903", + "diversity": { + "train": { + "mean_centered_corr": 0.08324653655290604, + "mean_pairwise_disagreement": 0.03557734342644149, + "member_std": 0.03504224866628647 + }, + "val": { + "mean_centered_corr": 0.09014386683702469, + "mean_pairwise_disagreement": 0.03127462845078635, + "member_std": 0.035533007234334946 + }, + "test": { + "mean_centered_corr": 0.11173245310783386, + "mean_pairwise_disagreement": 0.03804637777271872, + "member_std": 0.03772429749369621 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 28, + 17, + 30, + 22, + 20, + 10, + 29, + 28 + ], + "head_selection_time": "0:00:15.790811" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9894cb8d4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-best-head-evaluation/0/report.json @@ -0,0 +1,621 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9374104608021663, + "recall": 0.9816509223997231, + "f1-score": 0.9590207500067108, + "support": 181971.0 + }, + "1": { + "precision": 0.9010989010989011, + "recall": 0.7183640699898463, + "f1-score": 0.7994218893706477, + "support": 42349.0 + }, + "accuracy": 0.9319454350927246, + "macro avg": { + "precision": 0.9192546809505338, + "recall": 0.8500074961947847, + "f1-score": 0.8792213196886793, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9305552617923875, + "recall": 0.9319454350927246, + "f1-score": 0.9288903463464192, + "support": 224320.0 + }, + "cross-entropy": 0.15647591244364484, + "roc-auc": 0.9711251957175436, + "score": 0.9711251957175436 + }, + "val": { + "0": { + "precision": 0.9311434699175979, + "recall": 0.9854847380682158, + "f1-score": 0.9575437476421255, + "support": 16741.0 + }, + "1": { + "precision": 0.8995867768595042, + "recall": 0.6408595819841036, + "f1-score": 0.7484957882069796, + "support": 3397.0 + }, + "accuracy": 0.9273512761942596, + "macro avg": { + "precision": 0.915365123388551, + "recall": 0.8131721600261597, + "f1-score": 0.8530197679245526, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9258202955051267, + "recall": 0.9273512761942596, + "f1-score": 0.9222802697296619, + "support": 20138.0 + }, + "cross-entropy": 0.17757960670558423, + "roc-auc": 0.9577627701557911, + "score": 0.9577627701557911 + }, + "test": { + "0": { + "precision": 0.9243691471870755, + "recall": 0.9835703962881266, + "f1-score": 0.9530512971698113, + "support": 13147.0 + }, + "1": { + "precision": 0.9063313096270599, + "recall": 0.6639135959339263, + "f1-score": 0.7664099743307664, + "support": 3148.0 + }, + "accuracy": 0.9218165081313286, + "macro avg": { + "precision": 0.9153502284070677, + "recall": 0.8237419961110264, + "f1-score": 0.8597306357502889, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9208844517198199, + "recall": 0.9218165081313286, + "f1-score": 0.9169944156541737, + "support": 16295.0 + }, + "cross-entropy": 0.1822582690101261, + "roc-auc": 0.9622852537657216, + "score": 0.9622852537657216 + } + }, + "time": "0:03:33.494374", + "diversity": { + "train": { + "mean_centered_corr": 0.03106696531176567, + "mean_pairwise_disagreement": 0.027885827346302516, + "member_std": 0.025465095415711403 + }, + "val": { + "mean_centered_corr": 0.03693411499261856, + "mean_pairwise_disagreement": 0.02448668702084648, + "member_std": 0.02594522386789322 + }, + "test": { + "mean_centered_corr": 0.0499531626701355, + "mean_pairwise_disagreement": 0.0303318849043344, + "member_std": 0.027707822620868683 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:01.036071" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0.toml new file mode 100644 index 000000000..d4cdc115c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0.toml @@ -0,0 +1,45 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0/report.json new file mode 100644 index 000000000..f4fc30df1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-evaluation/0/report.json @@ -0,0 +1,617 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9422552559823976, + "recall": 0.983695204180886, + "f1-score": 0.9625294062294156, + "support": 181971.0 + }, + "1": { + "precision": 0.9136143946893379, + "recall": 0.7409620061866868, + "f1-score": 0.8182802007953582, + "support": 42349.0 + }, + "accuracy": 0.9378700071326677, + "macro avg": { + "precision": 0.9279348253358677, + "recall": 0.8623286051837864, + "f1-score": 0.8904048035123869, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9368481953774591, + "recall": 0.9378700071326677, + "f1-score": 0.9352968384649413, + "support": 224320.0 + }, + "cross-entropy": 0.1435463636983052, + "roc-auc": 0.976569916089968, + "score": 0.976569916089968 + }, + "val": { + "0": { + "precision": 0.9331481061465184, + "recall": 0.9830356609521534, + "f1-score": 0.9574424760741194, + "support": 16741.0 + }, + "1": { + "precision": 0.8864908073541167, + "recall": 0.6529290550485722, + "f1-score": 0.7519918630276318, + "support": 3397.0 + }, + "accuracy": 0.9273512761942596, + "macro avg": { + "precision": 0.9098194567503175, + "recall": 0.8179823580003629, + "f1-score": 0.8547171695508755, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9252776699563413, + "recall": 0.9273512761942596, + "f1-score": 0.9227858203725146, + "support": 20138.0 + }, + "cross-entropy": 0.17365686034344033, + "roc-auc": 0.9587449454385457, + "score": 0.9587449454385457 + }, + "test": { + "0": { + "precision": 0.9279746470757707, + "recall": 0.9799954362211911, + "f1-score": 0.9532758684473382, + "support": 13147.0 + }, + "1": { + "precision": 0.8909166321028619, + "recall": 0.6823379923761118, + "f1-score": 0.7728008634646519, + "support": 3148.0 + }, + "accuracy": 0.9224915618287818, + "macro avg": { + "precision": 0.9094456395893162, + "recall": 0.8311667142986514, + "f1-score": 0.863038365955995, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9208154797769234, + "recall": 0.9224915618287818, + "f1-score": 0.9184102461284983, + "support": 16295.0 + }, + "cross-entropy": 0.17782881823892016, + "roc-auc": 0.9627875642149869, + "score": 0.9627875642149869 + } + }, + "time": "0:03:33.494374", + "diversity": { + "train": { + "mean_centered_corr": 0.03106696531176567, + "mean_pairwise_disagreement": 0.027885827346302516, + "member_std": 0.025465095415711403 + }, + "val": { + "mean_centered_corr": 0.03693411499261856, + "mean_pairwise_disagreement": 0.02448668702084648, + "member_std": 0.02594522386789322 + }, + "test": { + "mean_centered_corr": 0.0499531626701355, + "mean_pairwise_disagreement": 0.0303318849043344, + "member_std": 0.027707822620868683 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..960e36c0c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,634 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9419323202817703, + "recall": 0.9817223623544411, + "f1-score": 0.961415820616798, + "support": 181971.0 + }, + "1": { + "precision": 0.9040447752582078, + "recall": 0.739946633922879, + "f1-score": 0.8138058199477997, + "support": 42349.0 + }, + "accuracy": 0.9360779243937233, + "macro avg": { + "precision": 0.9229885477699891, + "recall": 0.86083449813866, + "f1-score": 0.8876108202822988, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9347795936225207, + "recall": 0.9360779243937233, + "f1-score": 0.9335487739052636, + "support": 224320.0 + }, + "cross-entropy": 0.14666618497849432, + "roc-auc": 0.9751858546067549, + "score": 0.9751858546067549 + }, + "val": { + "0": { + "precision": 0.9337947466954104, + "recall": 0.9832148617167433, + "f1-score": 0.9578677839851024, + "support": 16741.0 + }, + "1": { + "precision": 0.8880923934687376, + "recall": 0.656461583750368, + "f1-score": 0.7549085985104943, + "support": 3397.0 + }, + "accuracy": 0.9280961366570663, + "macro avg": { + "precision": 0.910943570082074, + "recall": 0.8198382227335557, + "f1-score": 0.8563881912477984, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9260853965160973, + "recall": 0.9280961366570663, + "f1-score": 0.9236313973996797, + "support": 20138.0 + }, + "cross-entropy": 0.17262425691887265, + "roc-auc": 0.9594017599375494, + "score": 0.9594017599375494 + }, + "test": { + "0": { + "precision": 0.9273996258454454, + "recall": 0.980375751121929, + "f1-score": 0.9531521538177112, + "support": 13147.0 + }, + "1": { + "precision": 0.8923654568210263, + "recall": 0.6794790343074968, + "f1-score": 0.7715058611361587, + "support": 3148.0 + }, + "accuracy": 0.9222460877569807, + "macro avg": { + "precision": 0.9098825413332359, + "recall": 0.8299273927147128, + "f1-score": 0.862329007476935, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9206314414889636, + "recall": 0.9222460877569807, + "f1-score": 0.9180602526602073, + "support": 16295.0 + }, + "cross-entropy": 0.17716065755132018, + "roc-auc": 0.9632589952205967, + "score": 0.9632589952205967 + } + }, + "time": "0:03:33.494374", + "diversity": { + "train": { + "mean_centered_corr": 0.03106696531176567, + "mean_pairwise_disagreement": 0.027885827346302516, + "member_std": 0.025465095415711403 + }, + "val": { + "mean_centered_corr": 0.03693411499261856, + "mean_pairwise_disagreement": 0.02448668702084648, + "member_std": 0.02594522386789322 + }, + "test": { + "mean_centered_corr": 0.0499531626701355, + "mean_pairwise_disagreement": 0.0303318849043344, + "member_std": 0.027707822620868683 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 31, + 15, + 28, + 26, + 29, + 24, + 2, + 15, + 31, + 9, + 26, + 20, + 28 + ], + "head_selection_time": "0:00:14.082869" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-best-head-evaluation/0/report.json new file mode 100644 index 000000000..42c30294b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-best-head-evaluation/0/report.json @@ -0,0 +1,621 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2420, + "metrics": { + "train": { + "0": { + "precision": 0.9511390416339356, + "recall": 0.9648020838485253, + "f1-score": 0.9579218455024607, + "support": 181971.0 + }, + "1": { + "precision": 0.8388070970177426, + "recall": 0.7870315709934119, + "f1-score": 0.8120949271477998, + "support": 42349.0 + }, + "accuracy": 0.9312410841654779, + "macro avg": { + "precision": 0.8949730693258391, + "recall": 0.8759168274209685, + "f1-score": 0.8850083863251303, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9299320804956013, + "recall": 0.9312410841654779, + "f1-score": 0.9303914239377251, + "support": 224320.0 + }, + "cross-entropy": 0.15615645016500299, + "roc-auc": 0.9724991426058968, + "score": 0.9724991426058968 + }, + "val": { + "0": { + "precision": 0.9410138248847926, + "recall": 0.9758078967803596, + "f1-score": 0.958095070525791, + "support": 16741.0 + }, + "1": { + "precision": 0.8542116630669546, + "recall": 0.6985575507801001, + "f1-score": 0.768582995951417, + "support": 3397.0 + }, + "accuracy": 0.9290396265766213, + "macro avg": { + "precision": 0.8976127439758737, + "recall": 0.8371827237802298, + "f1-score": 0.863339033238604, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9263715096749804, + "recall": 0.9290396265766213, + "f1-score": 0.9261270241791256, + "support": 20138.0 + }, + "cross-entropy": 0.17528768630111824, + "roc-auc": 0.957813544584969, + "score": 0.957813544584969 + }, + "test": { + "0": { + "precision": 0.9342733348020856, + "recall": 0.9676732334372861, + "f1-score": 0.9506800179345389, + "support": 13147.0 + }, + "1": { + "precision": 0.8412994772218073, + "recall": 0.71569250317662, + "f1-score": 0.7734294541709578, + "support": 3148.0 + }, + "accuracy": 0.9189935563056152, + "macro avg": { + "precision": 0.8877864060119465, + "recall": 0.841682868306953, + "f1-score": 0.8620547360527484, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9163118924171383, + "recall": 0.9189935563056152, + "f1-score": 0.9164373192706695, + "support": 16295.0 + }, + "cross-entropy": 0.1803789882797406, + "roc-auc": 0.961490313471295, + "score": 0.961490313471295 + } + }, + "time": "0:03:47.889216", + "diversity": { + "train": { + "mean_centered_corr": 0.0506281740963459, + "mean_pairwise_disagreement": 0.03327413710540242, + "member_std": 0.03079444356262684 + }, + "val": { + "mean_centered_corr": 0.056434109807014465, + "mean_pairwise_disagreement": 0.030276074921749604, + "member_std": 0.03229342773556709 + }, + "test": { + "mean_centered_corr": 0.06734751909971237, + "mean_pairwise_disagreement": 0.037917825574834946, + "member_std": 0.03452758118510246 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.859514" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0.toml new file mode 100644 index 000000000..e655e5952 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0.toml @@ -0,0 +1,45 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0/report.json new file mode 100644 index 000000000..c86e0c52b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-evaluation/0/report.json @@ -0,0 +1,617 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2420, + "metrics": { + "train": { + "0": { + "precision": 0.9480090078156047, + "recall": 0.9831951244978595, + "f1-score": 0.965281524483674, + "support": 181971.0 + }, + "1": { + "precision": 0.9140890574518893, + "recall": 0.7683062173841176, + "f1-score": 0.8348814533511238, + "support": 42349.0 + }, + "accuracy": 0.942626604850214, + "macro avg": { + "precision": 0.931049032633747, + "recall": 0.8757506709409886, + "f1-score": 0.9000814889173989, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9416053167583962, + "recall": 0.942626604850214, + "f1-score": 0.9406635117679449, + "support": 224320.0 + }, + "cross-entropy": 0.13436786219631347, + "roc-auc": 0.9804692077039372, + "score": 0.9804692077039372 + }, + "val": { + "0": { + "precision": 0.9353075170842825, + "recall": 0.9810644525416642, + "f1-score": 0.957639718958631, + "support": 16741.0 + }, + "1": { + "precision": 0.8770364623739333, + "recall": 0.6655872828966736, + "f1-score": 0.7568200836820084, + "support": 3397.0 + }, + "accuracy": 0.9278478498361307, + "macro avg": { + "precision": 0.9061719897291078, + "recall": 0.8233258677191688, + "f1-score": 0.8572299013203197, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9254780020951546, + "recall": 0.9278478498361307, + "f1-score": 0.9237642446794232, + "support": 20138.0 + }, + "cross-entropy": 0.1724175473381701, + "roc-auc": 0.9589599740471012, + "score": 0.9589599740471012 + }, + "test": { + "0": { + "precision": 0.931104503589818, + "recall": 0.9765726021145509, + "f1-score": 0.9532967032967034, + "support": 13147.0 + }, + "1": { + "precision": 0.8770949720670391, + "recall": 0.6982210927573063, + "f1-score": 0.7775026529890343, + "support": 3148.0 + }, + "accuracy": 0.9227984044185333, + "macro avg": { + "precision": 0.9040997378284286, + "recall": 0.8373968474359286, + "f1-score": 0.8653996781428688, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9206705051096887, + "recall": 0.9227984044185333, + "f1-score": 0.919335385692006, + "support": 16295.0 + }, + "cross-entropy": 0.17668487055650292, + "roc-auc": 0.9628908822909436, + "score": 0.9628908822909436 + } + }, + "time": "0:03:47.889216", + "diversity": { + "train": { + "mean_centered_corr": 0.0506281740963459, + "mean_pairwise_disagreement": 0.03327413710540242, + "member_std": 0.03079444356262684 + }, + "val": { + "mean_centered_corr": 0.056434109807014465, + "mean_pairwise_disagreement": 0.030276074921749604, + "member_std": 0.03229342773556709 + }, + "test": { + "mean_centered_corr": 0.06734751909971237, + "mean_pairwise_disagreement": 0.037917825574834946, + "member_std": 0.03452758118510246 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c890b44c4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json @@ -0,0 +1,639 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2420, + "metrics": { + "train": { + "0": { + "precision": 0.9470743368373016, + "recall": 0.9808101290865028, + "f1-score": 0.9636470639051465, + "support": 181971.0 + }, + "1": { + "precision": 0.9026403100342933, + "recall": 0.7644808614134927, + "f1-score": 0.8278357369336197, + "support": 42349.0 + }, + "accuracy": 0.9399696861626248, + "macro avg": { + "precision": 0.9248573234357975, + "recall": 0.8726454952499978, + "f1-score": 0.8957414004193831, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9386857107625842, + "recall": 0.9399696861626248, + "f1-score": 0.9380074691926055, + "support": 224320.0 + }, + "cross-entropy": 0.13890143673129085, + "roc-auc": 0.9784781707405066, + "score": 0.9784781707405066 + }, + "val": { + "0": { + "precision": 0.9363594890510949, + "recall": 0.9808255181888776, + "f1-score": 0.9580768445313184, + "support": 16741.0 + }, + "1": { + "precision": 0.8766333589546502, + "recall": 0.6714748307329997, + "f1-score": 0.7604600766794466, + "support": 3397.0 + }, + "accuracy": 0.9286423676631245, + "macro avg": { + "precision": 0.9064964240028726, + "recall": 0.8261501744609387, + "f1-score": 0.8592684606053824, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9262845231191442, + "recall": 0.9286423676631245, + "f1-score": 0.9247416493583713, + "support": 20138.0 + }, + "cross-entropy": 0.17179269209939912, + "roc-auc": 0.9594844321379927, + "score": 0.9594844321379927 + }, + "test": { + "0": { + "precision": 0.9321799811005307, + "recall": 0.9754316574123374, + "f1-score": 0.9533154921201308, + "support": 13147.0 + }, + "1": { + "precision": 0.8727344365642238, + "recall": 0.7036213468869124, + "f1-score": 0.7791065775589167, + "support": 3148.0 + }, + "accuracy": 0.9229211414544338, + "macro avg": { + "precision": 0.9024572088323772, + "recall": 0.8395265021496249, + "f1-score": 0.8662110348395238, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9206958096245998, + "recall": 0.9229211414544338, + "f1-score": 0.9196604038698268, + "support": 16295.0 + }, + "cross-entropy": 0.1758390088286691, + "roc-auc": 0.9634382071404677, + "score": 0.9634382071404677 + } + }, + "time": "0:03:47.889216", + "diversity": { + "train": { + "mean_centered_corr": 0.0506281740963459, + "mean_pairwise_disagreement": 0.03327413710540242, + "member_std": 0.03079444356262684 + }, + "val": { + "mean_centered_corr": 0.056434109807014465, + "mean_pairwise_disagreement": 0.030276074921749604, + "member_std": 0.03229342773556709 + }, + "test": { + "mean_centered_corr": 0.06734751909971237, + "mean_pairwise_disagreement": 0.037917825574834946, + "member_std": 0.03452758118510246 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 2, + 0, + 28, + 7, + 26, + 18, + 29, + 24, + 15, + 4, + 7, + 27, + 19, + 31, + 0, + 2, + 28, + 11 + ], + "head_selection_time": "0:00:17.073344" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c778139f5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-best-head-evaluation/0/report.json @@ -0,0 +1,621 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2200, + "metrics": { + "train": { + "0": { + "precision": 0.9425672803027082, + "recall": 0.9746662929807497, + "f1-score": 0.9583480800456045, + "support": 181971.0 + }, + "1": { + "precision": 0.8724828501880947, + "recall": 0.7448109754657725, + "f1-score": 0.8036075973554477, + "support": 42349.0 + }, + "accuracy": 0.9312722895863053, + "macro avg": { + "precision": 0.9075250652454014, + "recall": 0.8597386342232611, + "f1-score": 0.880977838700526, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9293361572154945, + "recall": 0.9312722895863053, + "f1-score": 0.9291348814835261, + "support": 224320.0 + }, + "cross-entropy": 0.15513930110205976, + "roc-auc": 0.9715887850135724, + "score": 0.9715887850135724 + }, + "val": { + "0": { + "precision": 0.9368529428587771, + "recall": 0.9783764410728153, + "f1-score": 0.9571645628798504, + "support": 16741.0 + }, + "1": { + "precision": 0.863653483992467, + "recall": 0.6750073594347954, + "f1-score": 0.7577660277594184, + "support": 3397.0 + }, + "accuracy": 0.9272023041016982, + "macro avg": { + "precision": 0.900253213425622, + "recall": 0.8266919002538053, + "f1-score": 0.8574652953196344, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9245052140987783, + "recall": 0.9272023041016982, + "f1-score": 0.9235288083955865, + "support": 20138.0 + }, + "cross-entropy": 0.17668127284059484, + "roc-auc": 0.9576897428988642, + "score": 0.9576897428988642 + }, + "test": { + "0": { + "precision": 0.9302224476487211, + "recall": 0.9764965391344033, + "f1-score": 0.9527979812973133, + "support": 13147.0 + }, + "1": { + "precision": 0.876102646351243, + "recall": 0.6940914866581956, + "f1-score": 0.7745480326125488, + "support": 3148.0 + }, + "accuracy": 0.9219392451672292, + "macro avg": { + "precision": 0.903162546999982, + "recall": 0.8352940128962995, + "f1-score": 0.863673006954931, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9197671463609358, + "recall": 0.9219392451672292, + "f1-score": 0.9183622133648409, + "support": 16295.0 + }, + "cross-entropy": 0.18215599030843227, + "roc-auc": 0.9612223171103336, + "score": 0.9612223171103336 + } + }, + "time": "0:03:40.803387", + "diversity": { + "train": { + "mean_centered_corr": 0.03675549104809761, + "mean_pairwise_disagreement": 0.03015326247641618, + "member_std": 0.027854571118950844 + }, + "val": { + "mean_centered_corr": 0.03806319460272789, + "mean_pairwise_disagreement": 0.026045187400485042, + "member_std": 0.02860930748283863 + }, + "test": { + "mean_centered_corr": 0.04883061721920967, + "mean_pairwise_disagreement": 0.032468771342881744, + "member_std": 0.030201883986592293 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28 + ], + "head_selection_time": "0:00:01.169734" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0.toml new file mode 100644 index 000000000..2010232d3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0.toml @@ -0,0 +1,45 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.2 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0/report.json new file mode 100644 index 000000000..882db6358 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-evaluation/0/report.json @@ -0,0 +1,617 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2200, + "metrics": { + "train": { + "0": { + "precision": 0.942135079458505, + "recall": 0.9851954432299652, + "f1-score": 0.9631842346323393, + "support": 181971.0 + }, + "1": { + "precision": 0.9208392101551481, + "recall": 0.7399938605398002, + "f1-score": 0.8205705607415457, + "support": 42349.0 + }, + "accuracy": 0.9389042439372325, + "macro avg": { + "precision": 0.9314871448068266, + "recall": 0.8625946518848827, + "f1-score": 0.8918773976869425, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9381146676845755, + "recall": 0.9389042439372325, + "f1-score": 0.9362604361498089, + "support": 224320.0 + }, + "cross-entropy": 0.14122963803899807, + "roc-auc": 0.9777868186004166, + "score": 0.9777868186004166 + }, + "val": { + "0": { + "precision": 0.9316268294061141, + "recall": 0.9848276685980527, + "f1-score": 0.9574888204889948, + "support": 16741.0 + }, + "1": { + "precision": 0.8959442851290454, + "recall": 0.6438033559022667, + "f1-score": 0.7492291880781089, + "support": 3397.0 + }, + "accuracy": 0.9273016188300724, + "macro avg": { + "precision": 0.9137855572675797, + "recall": 0.8143155122501597, + "f1-score": 0.8533590042835518, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9256076813820202, + "recall": 0.9273016188300724, + "f1-score": 0.9223583223610883, + "support": 20138.0 + }, + "cross-entropy": 0.1736664220536342, + "roc-auc": 0.9587151577734279, + "score": 0.9587151577734279 + }, + "test": { + "0": { + "precision": 0.9260772926077293, + "recall": 0.9824294515859131, + "f1-score": 0.9534214217169853, + "support": 13147.0 + }, + "1": { + "precision": 0.901618398637138, + "recall": 0.6724904701397713, + "f1-score": 0.7703784570596798, + "support": 3148.0 + }, + "accuracy": 0.9225529303467321, + "macro avg": { + "precision": 0.9138478456224337, + "recall": 0.8274599608628421, + "f1-score": 0.8618999393883325, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9213521254877893, + "recall": 0.9225529303467321, + "f1-score": 0.9180597001618336, + "support": 16295.0 + }, + "cross-entropy": 0.17768434161277474, + "roc-auc": 0.9626366101271623, + "score": 0.9626366101271623 + } + }, + "time": "0:03:40.803387", + "diversity": { + "train": { + "mean_centered_corr": 0.03675549104809761, + "mean_pairwise_disagreement": 0.03015326247641618, + "member_std": 0.027854571118950844 + }, + "val": { + "mean_centered_corr": 0.03806319460272789, + "mean_pairwise_disagreement": 0.026045187400485042, + "member_std": 0.02860930748283863 + }, + "test": { + "mean_centered_corr": 0.04883061721920967, + "mean_pairwise_disagreement": 0.032468771342881744, + "member_std": 0.030201883986592293 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7186c0353 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json @@ -0,0 +1,634 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2200, + "metrics": { + "train": { + "0": { + "precision": 0.9428691849744482, + "recall": 0.9834863796978639, + "f1-score": 0.9627495756910655, + "support": 181971.0 + }, + "1": { + "precision": 0.9129237902057374, + "recall": 0.7439372830527285, + "f1-score": 0.8198129041491562, + "support": 42349.0 + }, + "accuracy": 0.9382623038516406, + "macro avg": { + "precision": 0.9278964875900928, + "recall": 0.8637118313752963, + "f1-score": 0.8912812399201109, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9372158436626608, + "recall": 0.9382623038516406, + "f1-score": 0.9357647990187744, + "support": 224320.0 + }, + "cross-entropy": 0.14301129796720108, + "roc-auc": 0.9769173245111457, + "score": 0.9769173245111457 + }, + "val": { + "0": { + "precision": 0.9320844416775143, + "recall": 0.9837524640105131, + "f1-score": 0.9572217378668991, + "support": 16741.0 + }, + "1": { + "precision": 0.8898339408667477, + "recall": 0.6467471298204298, + "f1-score": 0.7490623934538015, + "support": 3397.0 + }, + "accuracy": 0.9269043599165756, + "macro avg": { + "precision": 0.910959191272131, + "recall": 0.8152497969154715, + "f1-score": 0.8531420656603503, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9249573709031487, + "recall": 0.9269043599165756, + "f1-score": 0.922108156926821, + "support": 20138.0 + }, + "cross-entropy": 0.17259799258113656, + "roc-auc": 0.9594008895187635, + "score": 0.9594008895187635 + }, + "test": { + "0": { + "precision": 0.9257134662268751, + "recall": 0.9819730737050277, + "f1-score": 0.9530136935739859, + "support": 13147.0 + }, + "1": { + "precision": 0.8991060025542784, + "recall": 0.6709021601016518, + "f1-score": 0.768419137711479, + "support": 3148.0 + }, + "accuracy": 0.9218778766492789, + "macro avg": { + "precision": 0.9124097343905768, + "recall": 0.8264376169033398, + "f1-score": 0.8607164156427325, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9205732210202882, + "recall": 0.9218778766492789, + "f1-score": 0.9173522230704466, + "support": 16295.0 + }, + "cross-entropy": 0.1777880940764334, + "roc-auc": 0.9626779880984148, + "score": 0.9626779880984148 + } + }, + "time": "0:03:40.803387", + "diversity": { + "train": { + "mean_centered_corr": 0.03675549104809761, + "mean_pairwise_disagreement": 0.03015326247641618, + "member_std": 0.027854571118950844 + }, + "val": { + "mean_centered_corr": 0.03806319460272789, + "mean_pairwise_disagreement": 0.026045187400485042, + "member_std": 0.02860930748283863 + }, + "test": { + "mean_centered_corr": 0.04883061721920967, + "mean_pairwise_disagreement": 0.032468771342881744, + "member_std": 0.030201883986592293 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28, + 24, + 4, + 26, + 14, + 5, + 25, + 21, + 16, + 4, + 24, + 15, + 26, + 3 + ], + "head_selection_time": "0:00:14.459347" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1bc2f01c1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,631 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2200, + "metrics": { + "train": { + "0": { + "precision": 0.9395059644977903, + "recall": 0.9790076440751548, + "f1-score": 0.9588501398034946, + "support": 181971.0 + }, + "1": { + "precision": 0.8899071992622053, + "recall": 0.7291317386479019, + "f1-score": 0.801536724337093, + "support": 42349.0 + }, + "accuracy": 0.9318339871611983, + "macro avg": { + "precision": 0.9147065818799978, + "recall": 0.8540696913615284, + "f1-score": 0.8801934320702938, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9301422960377254, + "recall": 0.9318339871611983, + "f1-score": 0.9291512015385756, + "support": 224320.0 + }, + "cross-entropy": 0.15542274217664598, + "roc-auc": 0.9714271452985398, + "score": 0.9714271452985398 + }, + "val": { + "0": { + "precision": 0.9343040782663102, + "recall": 0.9811839197180575, + "f1-score": 0.9571703280694598, + "support": 16741.0 + }, + "1": { + "precision": 0.8768087602659367, + "recall": 0.6599941124521637, + "f1-score": 0.7531071548538797, + "support": 3397.0 + }, + "accuracy": 0.9270036746449498, + "macro avg": { + "precision": 0.9055564192661234, + "recall": 0.8205890160851106, + "f1-score": 0.8551387414616698, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9246054192511514, + "recall": 0.9270036746449498, + "f1-score": 0.9227477141349417, + "support": 20138.0 + }, + "cross-entropy": 0.17629531770563991, + "roc-auc": 0.9573917959108148, + "score": 0.9573917959108148 + }, + "test": { + "0": { + "precision": 0.9284371837501807, + "recall": 0.9769529170152886, + "f1-score": 0.9520773877914088, + "support": 13147.0 + }, + "1": { + "precision": 0.8768793173506705, + "recall": 0.6855146124523507, + "f1-score": 0.7694776252451417, + "support": 3148.0 + }, + "accuracy": 0.9206505062902731, + "macro avg": { + "precision": 0.9026582505504256, + "recall": 0.8312337647338197, + "f1-score": 0.8607775065182752, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9184768177835861, + "recall": 0.9206505062902731, + "f1-score": 0.9168012876075704, + "support": 16295.0 + }, + "cross-entropy": 0.18263783097893574, + "roc-auc": 0.9606087512633268, + "score": 0.9606087512633268 + } + }, + "time": "0:06:46.725623", + "diversity": { + "train": { + "mean_centered_corr": 0.030530203133821487, + "mean_pairwise_disagreement": 0.02995129905147025, + "member_std": 0.027598347514867783 + }, + "val": { + "mean_centered_corr": 0.03432288020849228, + "mean_pairwise_disagreement": 0.02602145999058112, + "member_std": 0.02860495075583458 + }, + "test": { + "mean_centered_corr": 0.04354812577366829, + "mean_pairwise_disagreement": 0.03329724633521068, + "member_std": 0.030326420441269875 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 7 + ], + "head_selection_time": "0:00:01.228380" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..0963236a7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0.toml @@ -0,0 +1,55 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..b4ca549ac --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/0/report.json @@ -0,0 +1,627 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2200, + "metrics": { + "train": { + "0": { + "precision": 0.9432183133443545, + "recall": 0.9845140159695775, + "f1-score": 0.9634238483069784, + "support": 181971.0 + }, + "1": { + "precision": 0.9180385085219009, + "recall": 0.7453304682519067, + "f1-score": 0.8227183276641774, + "support": 42349.0 + }, + "accuracy": 0.9393589514978602, + "macro avg": { + "precision": 0.9306284109331278, + "recall": 0.8649222421107421, + "f1-score": 0.8930710879855779, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9384646598385321, + "recall": 0.9393589514978602, + "f1-score": 0.9368602869049545, + "support": 224320.0 + }, + "cross-entropy": 0.1409082641420337, + "roc-auc": 0.9779900500547988, + "score": 0.9779900500547988 + }, + "val": { + "0": { + "precision": 0.9328840054372451, + "recall": 0.9838719311869064, + "f1-score": 0.9576997994011106, + "support": 16741.0 + }, + "1": { + "precision": 0.8912167606768735, + "recall": 0.6511627906976745, + "f1-score": 0.7525089300901514, + "support": 3397.0 + }, + "accuracy": 0.9277485351077565, + "macro avg": { + "precision": 0.9120503830570593, + "recall": 0.8175173609422904, + "f1-score": 0.855104364745631, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.925855321831575, + "recall": 0.9277485351077565, + "f1-score": 0.9230869588484575, + "support": 20138.0 + }, + "cross-entropy": 0.1728382451055005, + "roc-auc": 0.9589333075806602, + "score": 0.9589333075806602 + }, + "test": { + "0": { + "precision": 0.9278877293990644, + "recall": 0.9806800030425192, + "f1-score": 0.953553731232897, + "support": 13147.0 + }, + "1": { + "precision": 0.8941666666666667, + "recall": 0.681702668360864, + "f1-score": 0.7736121124729632, + "support": 3148.0 + }, + "accuracy": 0.9229211414544338, + "macro avg": { + "precision": 0.9110271980328655, + "recall": 0.8311913357016916, + "f1-score": 0.8635829218529301, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.921373221545024, + "recall": 0.9229211414544338, + "f1-score": 0.9187911527820672, + "support": 16295.0 + }, + "cross-entropy": 0.17708071781719004, + "roc-auc": 0.9627584002959787, + "score": 0.9627584002959787 + } + }, + "time": "0:06:46.725623", + "diversity": { + "train": { + "mean_centered_corr": 0.030530203133821487, + "mean_pairwise_disagreement": 0.02995129905147025, + "member_std": 0.027598347514867783 + }, + "val": { + "mean_centered_corr": 0.03432288020849228, + "mean_pairwise_disagreement": 0.02602145999058112, + "member_std": 0.02860495075583458 + }, + "test": { + "mean_centered_corr": 0.04354812577366829, + "mean_pairwise_disagreement": 0.03329724633521068, + "member_std": 0.030326420441269875 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ed9567004 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,642 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2200, + "metrics": { + "train": { + "0": { + "precision": 0.9421593289198092, + "recall": 0.9838435794714543, + "f1-score": 0.9625503705714893, + "support": 181971.0 + }, + "1": { + "precision": 0.9142807160767392, + "recall": 0.7404661267090132, + "f1-score": 0.8182446801570837, + "support": 42349.0 + }, + "accuracy": 0.9378967546362339, + "macro avg": { + "precision": 0.9282200224982742, + "recall": 0.8621548530902338, + "f1-score": 0.8903975253642865, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9368961719329549, + "recall": 0.9378967546362339, + "f1-score": 0.935307139101448, + "support": 224320.0 + }, + "cross-entropy": 0.14299189252771094, + "roc-auc": 0.9769227543613922, + "score": 0.9769227543613922 + }, + "val": { + "0": { + "precision": 0.9330086641372671, + "recall": 0.9841705991278896, + "f1-score": 0.957906976744186, + "support": 16741.0 + }, + "1": { + "precision": 0.8931020572811618, + "recall": 0.651751545481307, + "f1-score": 0.7535738597685501, + "support": 3397.0 + }, + "accuracy": 0.9280961366570663, + "macro avg": { + "precision": 0.9130553607092144, + "recall": 0.8179610723045982, + "f1-score": 0.855740418256368, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9262769756135711, + "recall": 0.9280961366570663, + "f1-score": 0.9234388270587042, + "support": 20138.0 + }, + "cross-entropy": 0.1721808588629276, + "roc-auc": 0.9595481045909983, + "score": 0.9595481045909983 + }, + "test": { + "0": { + "precision": 0.9277021674947793, + "recall": 0.9799193732410436, + "f1-score": 0.9530961012058888, + "support": 13147.0 + }, + "1": { + "precision": 0.8903654485049833, + "recall": 0.6810673443456162, + "f1-score": 0.7717782577393808, + "support": 3148.0 + }, + "accuracy": 0.9221847192390303, + "macro avg": { + "precision": 0.9090338079998813, + "recall": 0.8304933587933299, + "f1-score": 0.8624371794726349, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9204891578979779, + "recall": 0.9221847192390303, + "f1-score": 0.9180676525263818, + "support": 16295.0 + }, + "cross-entropy": 0.17638530532168833, + "roc-auc": 0.9631976229303886, + "score": 0.9631976229303886 + } + }, + "time": "0:06:46.725623", + "diversity": { + "train": { + "mean_centered_corr": 0.030530203133821487, + "mean_pairwise_disagreement": 0.02995129905147025, + "member_std": 0.027598347514867783 + }, + "val": { + "mean_centered_corr": 0.03432288020849228, + "mean_pairwise_disagreement": 0.02602145999058112, + "member_std": 0.02860495075583458 + }, + "test": { + "mean_centered_corr": 0.04354812577366829, + "mean_pairwise_disagreement": 0.03329724633521068, + "member_std": 0.030326420441269875 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 7, + 5, + 4, + 18, + 26, + 16, + 30, + 11, + 25, + 5, + 4, + 26 + ], + "head_selection_time": "0:00:13.901959" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..66229ec2b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,308 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9507099696263939, + "recall": 0.973567216754318, + "f1-score": 0.9620028399449387, + "support": 181971.0 + }, + "1": { + "precision": 0.873334386685627, + "recall": 0.7831117617889443, + "f1-score": 0.8257659698965427, + "support": 42349.0 + }, + "accuracy": 0.9376114479315264, + "macro avg": { + "precision": 0.9120221781560105, + "recall": 0.8783394892716312, + "f1-score": 0.8938844049207407, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9361023619143818, + "recall": 0.9376114479315264, + "f1-score": 0.9362829076621306, + "support": 224320.0 + }, + "cross-entropy": 0.14378673331190528, + "roc-auc": 0.9763699988763426, + "score": 0.9763699988763426 + }, + "val": { + "0": { + "precision": 0.9398387190346348, + "recall": 0.9676841287856162, + "f1-score": 0.9535581847077521, + "support": 16741.0 + }, + "1": { + "precision": 0.8135125818683213, + "recall": 0.694730644686488, + "f1-score": 0.7494442680215941, + "support": 3397.0 + }, + "accuracy": 0.9216406793127421, + "macro avg": { + "precision": 0.8766756504514781, + "recall": 0.8312073867360521, + "f1-score": 0.8515012263646731, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9185292599049314, + "recall": 0.9216406793127421, + "f1-score": 0.9191270110567998, + "support": 20138.0 + }, + "cross-entropy": 0.18378915635489101, + "roc-auc": 0.9551748656394307, + "score": 0.9551748656394307 + }, + "test": { + "0": { + "precision": 0.9405636336381352, + "recall": 0.9545143378717578, + "f1-score": 0.9474876363773642, + "support": 13147.0 + }, + "1": { + "precision": 0.7974940738232306, + "recall": 0.7480940279542567, + "f1-score": 0.7720045894115719, + "support": 3148.0 + }, + "accuracy": 0.9146363915311445, + "macro avg": { + "precision": 0.869028853730683, + "recall": 0.8513041829130072, + "f1-score": 0.8597461128944681, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9129242979954644, + "recall": 0.9146363915311445, + "f1-score": 0.9135864009156696, + "support": 16295.0 + }, + "cross-entropy": 0.1875048907820305, + "roc-auc": 0.9597148541915197, + "score": 0.9597148541915197 + } + }, + "time": "0:05:30.729154", + "diversity": { + "train": { + "mean_centered_corr": 0.08804295212030411, + "mean_pairwise_disagreement": 0.03950866022329851, + "member_std": 0.03889348357915878 + }, + "val": { + "mean_centered_corr": 0.0962420254945755, + "mean_pairwise_disagreement": 0.03718395490470592, + "member_std": 0.042661890387535095 + }, + "test": { + "mean_centered_corr": 0.11283686012029648, + "mean_pairwise_disagreement": 0.04520397608607429, + "member_std": 0.04494225233793259 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:01.037395" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0.toml new file mode 100644 index 000000000..e5a6651c3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0.toml @@ -0,0 +1,57 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0/report.json new file mode 100644 index 000000000..31058d82c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/0/report.json @@ -0,0 +1,304 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9469197964827833, + "recall": 0.9859318243016745, + "f1-score": 0.966032107560057, + "support": 181971.0 + }, + "1": { + "precision": 0.9265465396533915, + "recall": 0.7625209568112589, + "f1-score": 0.8365694744886725, + "support": 42349.0 + }, + "accuracy": 0.9437544579172611, + "macro avg": { + "precision": 0.9367331680680875, + "recall": 0.8742263905564667, + "f1-score": 0.9013007910243647, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9430735631845134, + "recall": 0.9437544579172611, + "f1-score": 0.9415910722179561, + "support": 224320.0 + }, + "cross-entropy": 0.13682316815786993, + "roc-auc": 0.9807455131029597, + "score": 0.9807455131029597 + }, + "val": { + "0": { + "precision": 0.9328421649192405, + "recall": 0.9832148617167433, + "f1-score": 0.9573663700343162, + "support": 16741.0 + }, + "1": { + "precision": 0.887284396309667, + "recall": 0.6511627906976745, + "f1-score": 0.7511035653650254, + "support": 3397.0 + }, + "accuracy": 0.9272023041016982, + "macro avg": { + "precision": 0.9100632806144537, + "recall": 0.8171888262072089, + "f1-score": 0.8542349676996708, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9251572041502107, + "recall": 0.9272023041016982, + "f1-score": 0.922572708922906, + "support": 20138.0 + }, + "cross-entropy": 0.17538838012996275, + "roc-auc": 0.9580893002900325, + "score": 0.9580893002900325 + }, + "test": { + "0": { + "precision": 0.9285301344124873, + "recall": 0.9773332319160265, + "f1-score": 0.9523068371317398, + "support": 13147.0 + }, + "1": { + "precision": 0.8787138787138787, + "recall": 0.6858322744599746, + "f1-score": 0.7703835860838537, + "support": 3148.0 + }, + "accuracy": 0.9210187173979748, + "macro avg": { + "precision": 0.903622006563183, + "recall": 0.8315827531880006, + "f1-score": 0.8613452116077968, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9189062268985738, + "recall": 0.9210187173979748, + "f1-score": 0.9171614309151859, + "support": 16295.0 + }, + "cross-entropy": 0.17996585840914717, + "roc-auc": 0.9618081204528327, + "score": 0.9618081204528327 + } + }, + "time": "0:05:30.729154", + "diversity": { + "train": { + "mean_centered_corr": 0.08804295212030411, + "mean_pairwise_disagreement": 0.03950866022329851, + "member_std": 0.03889348357915878 + }, + "val": { + "mean_centered_corr": 0.0962420254945755, + "mean_pairwise_disagreement": 0.03718395490470592, + "member_std": 0.042661890387535095 + }, + "test": { + "mean_centered_corr": 0.11283686012029648, + "mean_pairwise_disagreement": 0.04520397608607429, + "member_std": 0.04494225233793259 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..58a2a2d41 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,317 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9477218327013917, + "recall": 0.9834753889355996, + "f1-score": 0.9652676459721634, + "support": 181971.0 + }, + "1": { + "precision": 0.9152575808815241, + "recall": 0.7668894188764788, + "f1-score": 0.8345303406010304, + "support": 42349.0 + }, + "accuracy": 0.9425864835948645, + "macro avg": { + "precision": 0.9314897067914578, + "recall": 0.8751824039060392, + "f1-score": 0.8998989932865968, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9415929605530341, + "recall": 0.9425864835948645, + "f1-score": 0.9405859673649856, + "support": 224320.0 + }, + "cross-entropy": 0.13852674322862857, + "roc-auc": 0.9799226934193556, + "score": 0.9799226934193556 + }, + "val": { + "0": { + "precision": 0.9338218586684844, + "recall": 0.9819604563646138, + "f1-score": 0.9572863590042219, + "support": 16741.0 + }, + "1": { + "precision": 0.8808208366219415, + "recall": 0.6570503385340006, + "f1-score": 0.7526555386949925, + "support": 3397.0 + }, + "accuracy": 0.9271526467375112, + "macro avg": { + "precision": 0.907321347645213, + "recall": 0.8195053974493072, + "f1-score": 0.8549709488496071, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9248813247578624, + "recall": 0.9271526467375112, + "f1-score": 0.9227679909145182, + "support": 20138.0 + }, + "cross-entropy": 0.17399362918195638, + "roc-auc": 0.9588471624971818, + "score": 0.9588471624971818 + }, + "test": { + "0": { + "precision": 0.9311070780399274, + "recall": 0.9755837833726325, + "f1-score": 0.9528266844959513, + "support": 13147.0 + }, + "1": { + "precision": 0.8726190476190476, + "recall": 0.6985387547649301, + "f1-score": 0.7759350741002117, + "support": 3148.0 + }, + "accuracy": 0.9220619822031298, + "macro avg": { + "precision": 0.9018630628294875, + "recall": 0.8370612690687813, + "f1-score": 0.8643808792980815, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9198078868914199, + "recall": 0.9220619822031298, + "f1-score": 0.9186533313492322, + "support": 16295.0 + }, + "cross-entropy": 0.17933181253710723, + "roc-auc": 0.962094141903753, + "score": 0.962094141903753 + } + }, + "time": "0:05:30.729154", + "diversity": { + "train": { + "mean_centered_corr": 0.08804295212030411, + "mean_pairwise_disagreement": 0.03950866022329851, + "member_std": 0.03889348357915878 + }, + "val": { + "mean_centered_corr": 0.0962420254945755, + "mean_pairwise_disagreement": 0.03718395490470592, + "member_std": 0.042661890387535095 + }, + "test": { + "mean_centered_corr": 0.11283686012029648, + "mean_pairwise_disagreement": 0.04520397608607429, + "member_std": 0.04494225233793259 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 4, + 7, + 22, + 16, + 23, + 5, + 30, + 17, + 7 + ], + "head_selection_time": "0:00:11.116484" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..fb3e78cdb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-best-head-evaluation/0/report.json @@ -0,0 +1,272 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9505616894590861, + "recall": 0.9755675354864236, + "f1-score": 0.9629022940967485, + "support": 181971.0 + }, + "1": { + "precision": 0.8816356956498589, + "recall": 0.7819783229828331, + "f1-score": 0.8288220645468083, + "support": 42349.0 + }, + "accuracy": 0.93902014978602, + "macro avg": { + "precision": 0.9160986925544725, + "recall": 0.8787729292346284, + "f1-score": 0.8958621793217785, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9375492656367478, + "recall": 0.93902014978602, + "f1-score": 0.9375895103894981, + "support": 224320.0 + }, + "cross-entropy": 0.14190517817089304, + "roc-auc": 0.9769881887024199, + "score": 0.9769881887024199 + }, + "val": { + "0": { + "precision": 0.9410778304614317, + "recall": 0.9721641479003644, + "f1-score": 0.9563684442485677, + "support": 16741.0 + }, + "1": { + "precision": 0.8361462728551337, + "recall": 0.7000294377391816, + "f1-score": 0.7620573626021471, + "support": 3397.0 + }, + "accuracy": 0.9262588141821432, + "macro avg": { + "precision": 0.8886120516582827, + "recall": 0.836096792819773, + "f1-score": 0.8592129034253574, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9233773387945037, + "recall": 0.9262588141821432, + "f1-score": 0.9235908722775235, + "support": 20138.0 + }, + "cross-entropy": 0.17980444186714722, + "roc-auc": 0.9561281764285071, + "score": 0.9561281764285071 + }, + "test": { + "0": { + "precision": 0.937955119631446, + "recall": 0.9601429984026775, + "f1-score": 0.9489193760571322, + "support": 13147.0 + }, + "1": { + "precision": 0.8152978498413818, + "recall": 0.7347522236340533, + "f1-score": 0.7729323308270677, + "support": 3148.0 + }, + "accuracy": 0.9166001841055539, + "macro avg": { + "precision": 0.8766264847364138, + "recall": 0.8474476110183654, + "f1-score": 0.8609258534420999, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.914259195403209, + "recall": 0.9166001841055539, + "f1-score": 0.9149207741311277, + "support": 16295.0 + }, + "cross-entropy": 0.18582539957016014, + "roc-auc": 0.9607344074031798, + "score": 0.9607344074031798 + } + }, + "time": "0:05:03.728563", + "diversity": { + "train": { + "mean_centered_corr": 0.051523078233003616, + "mean_pairwise_disagreement": 0.03397294259928213, + "member_std": 0.03101634979248047 + }, + "val": { + "mean_centered_corr": 0.06029462069272995, + "mean_pairwise_disagreement": 0.032282392619954574, + "member_std": 0.03389967605471611 + }, + "test": { + "mean_centered_corr": 0.07432211190462112, + "mean_pairwise_disagreement": 0.04083245899692168, + "member_std": 0.03589051589369774 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.792824" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0.toml new file mode 100644 index 000000000..1c9bfed00 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0/report.json new file mode 100644 index 000000000..aa3e33961 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/0/report.json @@ -0,0 +1,268 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.947970671011702, + "recall": 0.9847338312148639, + "f1-score": 0.9660026037806032, + "support": 181971.0 + }, + "1": { + "precision": 0.921285277116627, + "recall": 0.7677631112895228, + "f1-score": 0.8375471722414704, + "support": 42349.0 + }, + "accuracy": 0.9437722895863053, + "macro avg": { + "precision": 0.9346279740641645, + "recall": 0.8762484712521934, + "f1-score": 0.9017748880110368, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9429327798470153, + "recall": 0.9437722895863053, + "f1-score": 0.9417517163418964, + "support": 224320.0 + }, + "cross-entropy": 0.13293877063076864, + "roc-auc": 0.9811870501530091, + "score": 0.9811870501530091 + }, + "val": { + "0": { + "precision": 0.9362066016760732, + "recall": 0.9809449853652709, + "f1-score": 0.9580537891604924, + "support": 16741.0 + }, + "1": { + "precision": 0.8771659607239122, + "recall": 0.6705916985575507, + "f1-score": 0.7600934267600934, + "support": 3397.0 + }, + "accuracy": 0.9285927102989373, + "macro avg": { + "precision": 0.9066862811999927, + "recall": 0.8257683419614108, + "f1-score": 0.859073607960293, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9262472682112558, + "recall": 0.9285927102989373, + "f1-score": 0.9246606343748058, + "support": 20138.0 + }, + "cross-entropy": 0.17347812058673698, + "roc-auc": 0.958366515485181, + "score": 0.958366515485181 + }, + "test": { + "0": { + "precision": 0.9325041821223362, + "recall": 0.9752034684718948, + "f1-score": 0.9533759666864962, + "support": 13147.0 + }, + "1": { + "precision": 0.8719560094265515, + "recall": 0.7052096569250318, + "f1-score": 0.779768177028451, + "support": 3148.0 + }, + "accuracy": 0.9230438784903344, + "macro avg": { + "precision": 0.9022300957744438, + "recall": 0.8402065626984633, + "f1-score": 0.8665720718574736, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9208069960133255, + "recall": 0.9230438784903344, + "f1-score": 0.9198370086107965, + "support": 16295.0 + }, + "cross-entropy": 0.17737713039078654, + "roc-auc": 0.9625169220800972, + "score": 0.9625169220800972 + } + }, + "time": "0:05:03.728563", + "diversity": { + "train": { + "mean_centered_corr": 0.051523078233003616, + "mean_pairwise_disagreement": 0.03397294259928213, + "member_std": 0.03101634979248047 + }, + "val": { + "mean_centered_corr": 0.06029462069272995, + "mean_pairwise_disagreement": 0.032282392619954574, + "member_std": 0.03389967605471611 + }, + "test": { + "mean_centered_corr": 0.07432211190462112, + "mean_pairwise_disagreement": 0.04083245899692168, + "member_std": 0.03589051589369774 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..812f0329f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,285 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9463750527648797, + "recall": 0.9856350737205379, + "f1-score": 0.9656061654252728, + "support": 181971.0 + }, + "1": { + "precision": 0.9248850574712644, + "recall": 0.7600179461144301, + "f1-score": 0.8343854100506811, + "support": 42349.0 + }, + "accuracy": 0.9430411911554921, + "macro avg": { + "precision": 0.9356300551180721, + "recall": 0.872826509917484, + "f1-score": 0.899995787737977, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9423179922678696, + "recall": 0.9430411911554921, + "f1-score": 0.9408332170953934, + "support": 224320.0 + }, + "cross-entropy": 0.13463392233996885, + "roc-auc": 0.9802844103471857, + "score": 0.9802844103471857 + }, + "val": { + "0": { + "precision": 0.9344280925052559, + "recall": 0.9823188578937937, + "f1-score": 0.9577751892836343, + "support": 16741.0 + }, + "1": { + "precision": 0.8834186687672312, + "recall": 0.66028848984398, + "f1-score": 0.7557277628032345, + "support": 3397.0 + }, + "accuracy": 0.927996821928692, + "macro avg": { + "precision": 0.9089233806362436, + "recall": 0.8213036738688868, + "f1-score": 0.8567514760434344, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9258235134786361, + "recall": 0.927996821928692, + "f1-score": 0.9236926037362155, + "support": 20138.0 + }, + "cross-entropy": 0.17320355151838052, + "roc-auc": 0.9589766350935588, + "score": 0.9589766350935588 + }, + "test": { + "0": { + "precision": 0.9308371958285052, + "recall": 0.9776374838366167, + "f1-score": 0.9536635132628455, + "support": 13147.0 + }, + "1": { + "precision": 0.8817852834740652, + "recall": 0.6966327827191868, + "f1-score": 0.7783496007098492, + "support": 3148.0 + }, + "accuracy": 0.9233507210800859, + "macro avg": { + "precision": 0.9063112396512851, + "recall": 0.8371351332779018, + "f1-score": 0.8660065569863473, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9213609503488013, + "recall": 0.9233507210800859, + "f1-score": 0.9197949525560746, + "support": 16295.0 + }, + "cross-entropy": 0.17676394803557405, + "roc-auc": 0.962915757398333, + "score": 0.962915757398333 + } + }, + "time": "0:05:03.728563", + "diversity": { + "train": { + "mean_centered_corr": 0.051523078233003616, + "mean_pairwise_disagreement": 0.03397294259928213, + "member_std": 0.03101634979248047 + }, + "val": { + "mean_centered_corr": 0.06029462069272995, + "mean_pairwise_disagreement": 0.032282392619954574, + "member_std": 0.03389967605471611 + }, + "test": { + "mean_centered_corr": 0.07432211190462112, + "mean_pairwise_disagreement": 0.04083245899692168, + "member_std": 0.03589051589369774 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 4, + 26, + 2, + 24, + 9, + 12, + 7, + 28, + 15, + 26, + 17, + 2, + 3 + ], + "head_selection_time": "0:00:13.310784" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3d42603b9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-best-head-evaluation/0/report.json @@ -0,0 +1,272 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9320793769889524, + "recall": 0.9898775079545642, + "f1-score": 0.9601093740838854, + "support": 181971.0 + }, + "1": { + "precision": 0.9407049734427813, + "recall": 0.6900517131455288, + "f1-score": 0.7961151823902798, + "support": 42349.0 + }, + "accuracy": 0.9332738944365193, + "macro avg": { + "precision": 0.9363921752158668, + "recall": 0.8399646105500465, + "f1-score": 0.8781122782370826, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9337077890084923, + "recall": 0.9332738944365193, + "f1-score": 0.9291491831778916, + "support": 224320.0 + }, + "cross-entropy": 0.14914181629256673, + "roc-auc": 0.9749246612138766, + "score": 0.9749246612138766 + }, + "val": { + "0": { + "precision": 0.9254064927082751, + "recall": 0.9893076877128009, + "f1-score": 0.9562907789133321, + "support": 16741.0 + }, + "1": { + "precision": 0.9201249442213297, + "recall": 0.6070061819252281, + "f1-score": 0.7314650585313941, + "support": 3397.0 + }, + "accuracy": 0.9248187506207171, + "macro avg": { + "precision": 0.9227657184648024, + "recall": 0.7981569348190145, + "f1-score": 0.8438779187223631, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9245155690708656, + "recall": 0.9248187506207171, + "f1-score": 0.9183658125741999, + "support": 20138.0 + }, + "cross-entropy": 0.18253383914635615, + "roc-auc": 0.9561005955827354, + "score": 0.9561005955827354 + }, + "test": { + "0": { + "precision": 0.9195605953224664, + "recall": 0.9869171674146193, + "f1-score": 0.952049014931944, + "support": 13147.0 + }, + "1": { + "precision": 0.9212814645308924, + "recall": 0.6394536213468869, + "f1-score": 0.7549221826364148, + "support": 3148.0 + }, + "accuracy": 0.919791347038969, + "macro avg": { + "precision": 0.9204210299266794, + "recall": 0.8131853943807531, + "f1-score": 0.8534855987841794, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9198930467657388, + "recall": 0.919791347038969, + "f1-score": 0.9139664578244677, + "support": 16295.0 + }, + "cross-entropy": 0.1878335545651875, + "roc-auc": 0.9607496659076156, + "score": 0.9607496659076156 + } + }, + "time": "0:04:50.502999", + "diversity": { + "train": { + "mean_centered_corr": 0.055189475417137146, + "mean_pairwise_disagreement": 0.03156465166409737, + "member_std": 0.02923482470214367 + }, + "val": { + "mean_centered_corr": 0.06409875303506851, + "mean_pairwise_disagreement": 0.02878024694126655, + "member_std": 0.03127071633934975 + }, + "test": { + "mean_centered_corr": 0.07738936692476273, + "mean_pairwise_disagreement": 0.03777504478911996, + "member_std": 0.03404448181390762 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.834549" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0.toml new file mode 100644 index 000000000..a0ce28efa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0/report.json new file mode 100644 index 000000000..779546f37 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/0/report.json @@ -0,0 +1,268 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9447991304623112, + "recall": 0.9840249270488155, + "f1-score": 0.9640131683432168, + "support": 181971.0 + }, + "1": { + "precision": 0.9164511122607346, + "recall": 0.7529575668846962, + "f1-score": 0.8266984690769091, + "support": 42349.0 + }, + "accuracy": 0.9404021041369472, + "macro avg": { + "precision": 0.9306251213615229, + "recall": 0.8684912469667558, + "f1-score": 0.895355818710063, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9394473552179345, + "recall": 0.9404021041369472, + "f1-score": 0.9380897544736159, + "support": 224320.0 + }, + "cross-entropy": 0.14008234966516878, + "roc-auc": 0.978457814057529, + "score": 0.978457814057529 + }, + "val": { + "0": { + "precision": 0.9334657961964234, + "recall": 0.9821993907174004, + "f1-score": 0.9572127139364304, + "support": 16741.0 + }, + "1": { + "precision": 0.8818866428854538, + "recall": 0.6549896967912864, + "f1-score": 0.7516891891891891, + "support": 3397.0 + }, + "accuracy": 0.9270036746449498, + "macro avg": { + "precision": 0.9076762195409387, + "recall": 0.8185945437543434, + "f1-score": 0.8544509515628098, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9247651117293779, + "recall": 0.9270036746449498, + "f1-score": 0.9225437590468496, + "support": 20138.0 + }, + "cross-entropy": 0.17400317835615806, + "roc-auc": 0.9581563313286563, + "score": 0.9581563313286563 + }, + "test": { + "0": { + "precision": 0.9298753984352361, + "recall": 0.9763444131741081, + "f1-score": 0.9525435048792252, + "support": 13147.0 + }, + "1": { + "precision": 0.8751505419510237, + "recall": 0.6925031766200762, + "f1-score": 0.7731867352367441, + "support": 3148.0 + }, + "accuracy": 0.9215096655415772, + "macro avg": { + "precision": 0.9025129701931299, + "recall": 0.8344237948970922, + "f1-score": 0.8628651200579847, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9193032076888538, + "recall": 0.9215096655415772, + "f1-score": 0.9178939123149706, + "support": 16295.0 + }, + "cross-entropy": 0.17809609825505548, + "roc-auc": 0.96205585670933, + "score": 0.96205585670933 + } + }, + "time": "0:04:50.502999", + "diversity": { + "train": { + "mean_centered_corr": 0.055189475417137146, + "mean_pairwise_disagreement": 0.03156465166409737, + "member_std": 0.02923482470214367 + }, + "val": { + "mean_centered_corr": 0.06409875303506851, + "mean_pairwise_disagreement": 0.02878024694126655, + "member_std": 0.03127071633934975 + }, + "test": { + "mean_centered_corr": 0.07738936692476273, + "mean_pairwise_disagreement": 0.03777504478911996, + "member_std": 0.03404448181390762 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..59be8f2e4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,281 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9408421041602272, + "recall": 0.9868990113809343, + "f1-score": 0.963320370122033, + "support": 181971.0 + }, + "1": { + "precision": 0.9287102658413325, + "recall": 0.733358520862358, + "f1-score": 0.8195540308747856, + "support": 42349.0 + }, + "accuracy": 0.9390335235378031, + "macro avg": { + "precision": 0.9347761850007799, + "recall": 0.8601287661216461, + "f1-score": 0.8914372004984092, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9385517545660453, + "recall": 0.9390335235378031, + "f1-score": 0.9361789618624856, + "support": 224320.0 + }, + "cross-entropy": 0.1413648202791101, + "roc-auc": 0.9778507602646593, + "score": 0.9778507602646593 + }, + "val": { + "0": { + "precision": 0.9304607749139925, + "recall": 0.9854847380682158, + "f1-score": 0.957182640983987, + "support": 16741.0 + }, + "1": { + "precision": 0.8990444536767761, + "recall": 0.6370326758904916, + "f1-score": 0.7456926257753274, + "support": 3397.0 + }, + "accuracy": 0.9267057304598272, + "macro avg": { + "precision": 0.9147526142953843, + "recall": 0.8112587069793538, + "f1-score": 0.8514376333796572, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9251612792717825, + "recall": 0.9267057304598272, + "f1-score": 0.9215072222897861, + "support": 20138.0 + }, + "cross-entropy": 0.17378940292970774, + "roc-auc": 0.95870368407125, + "score": 0.95870368407125 + }, + "test": { + "0": { + "precision": 0.9256684108666046, + "recall": 0.982277325625618, + "f1-score": 0.9531330725514798, + "support": 13147.0 + }, + "1": { + "precision": 0.9005972696245734, + "recall": 0.670584498094028, + "f1-score": 0.7687545520757465, + "support": 3148.0 + }, + "accuracy": 0.9220619822031298, + "macro avg": { + "precision": 0.9131328402455889, + "recall": 0.826430911859823, + "f1-score": 0.8609438123136132, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9208249648629276, + "recall": 0.9220619822031298, + "f1-score": 0.9175133375126576, + "support": 16295.0 + }, + "cross-entropy": 0.17801794809138968, + "roc-auc": 0.962465432178352, + "score": 0.962465432178352 + } + }, + "time": "0:04:50.502999", + "diversity": { + "train": { + "mean_centered_corr": 0.055189475417137146, + "mean_pairwise_disagreement": 0.03156465166409737, + "member_std": 0.02923482470214367 + }, + "val": { + "mean_centered_corr": 0.06409875303506851, + "mean_pairwise_disagreement": 0.02878024694126655, + "member_std": 0.03127071633934975 + }, + "test": { + "mean_centered_corr": 0.07738936692476273, + "mean_pairwise_disagreement": 0.03777504478911996, + "member_std": 0.03404448181390762 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 13, + 28, + 5, + 30, + 12, + 21, + 4, + 17, + 29, + 31 + ], + "head_selection_time": "0:00:10.965687" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5e67db395 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-best-head-evaluation/0/report.json @@ -0,0 +1,272 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9545321160743397, + "recall": 0.9652801820070231, + "f1-score": 0.9598760625261277, + "support": 181971.0 + }, + "1": { + "precision": 0.8432258064516129, + "recall": 0.8024274481097546, + "f1-score": 0.8223208992244311, + "support": 42349.0 + }, + "accuracy": 0.934535485021398, + "macro avg": { + "precision": 0.8988789612629764, + "recall": 0.8838538150583889, + "f1-score": 0.8910984808752793, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9335187828619073, + "recall": 0.934535485021398, + "f1-score": 0.9339072518509157, + "support": 224320.0 + }, + "cross-entropy": 0.1498442336366005, + "roc-auc": 0.9747257011404723, + "score": 0.9747257011404723 + }, + "val": { + "0": { + "precision": 0.9430700273589848, + "recall": 0.9677438623738128, + "f1-score": 0.955247641509434, + "support": 16741.0 + }, + "1": { + "precision": 0.8175059141601893, + "recall": 0.7120989108036503, + "f1-score": 0.7611705475141598, + "support": 3397.0 + }, + "accuracy": 0.9246201211639686, + "macro avg": { + "precision": 0.880287970759587, + "recall": 0.8399213865887316, + "f1-score": 0.8582090945117968, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9218891110546691, + "recall": 0.9246201211639686, + "f1-score": 0.9225095399947877, + "support": 20138.0 + }, + "cross-entropy": 0.18072654373317865, + "roc-auc": 0.9561512908829329, + "score": 0.9561512908829329 + }, + "test": { + "0": { + "precision": 0.938474098263322, + "recall": 0.9618163839659238, + "f1-score": 0.9500018782164457, + "support": 13147.0 + }, + "1": { + "precision": 0.8220489188231124, + "recall": 0.7366581956797967, + "f1-score": 0.7770145753057464, + "support": 3148.0 + }, + "accuracy": 0.9183185026081621, + "macro avg": { + "precision": 0.8802615085432172, + "recall": 0.8492372898228603, + "f1-score": 0.8635082267610961, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9159821396945721, + "recall": 0.9183185026081621, + "f1-score": 0.9165827907931329, + "support": 16295.0 + }, + "cross-entropy": 0.18568286621420937, + "roc-auc": 0.9606910602029306, + "score": 0.9606910602029306 + } + }, + "time": "0:05:03.738226", + "diversity": { + "train": { + "mean_centered_corr": 0.049995824694633484, + "mean_pairwise_disagreement": 0.03397909919872533, + "member_std": 0.031108465045690536 + }, + "val": { + "mean_centered_corr": 0.05969587340950966, + "mean_pairwise_disagreement": 0.03266793800197989, + "member_std": 0.03460051864385605 + }, + "test": { + "mean_centered_corr": 0.07376471161842346, + "mean_pairwise_disagreement": 0.04182474833958566, + "member_std": 0.03703681379556656 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 17 + ], + "head_selection_time": "0:00:00.955041" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0.toml new file mode 100644 index 000000000..016b99992 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0/report.json new file mode 100644 index 000000000..382e4333a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/0/report.json @@ -0,0 +1,268 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9496102021429594, + "recall": 0.9833215182638992, + "f1-score": 0.9661718889206862, + "support": 181971.0 + }, + "1": { + "precision": 0.9154336983476832, + "recall": 0.7757916361661432, + "f1-score": 0.8398476443671873, + "support": 42349.0 + }, + "accuracy": 0.9441422967189729, + "macro avg": { + "precision": 0.9325219502453213, + "recall": 0.8795565772150212, + "f1-score": 0.9030097666439367, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9431580767897758, + "recall": 0.9441422967189729, + "f1-score": 0.9423233447311528, + "support": 224320.0 + }, + "cross-entropy": 0.13329761494760606, + "roc-auc": 0.9810524262916843, + "score": 0.9810524262916843 + }, + "val": { + "0": { + "precision": 0.936472602739726, + "recall": 0.9800489815423212, + "f1-score": 0.9577653892180613, + "support": 16741.0 + }, + "1": { + "precision": 0.8724216959511077, + "recall": 0.6723579629084486, + "f1-score": 0.7594347464671655, + "support": 3397.0 + }, + "accuracy": 0.9281457940212533, + "macro avg": { + "precision": 0.9044471493454169, + "recall": 0.8262034722253849, + "f1-score": 0.8586000678426133, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9256681072406231, + "recall": 0.9281457940212533, + "f1-score": 0.9243097732966793, + "support": 20138.0 + }, + "cross-entropy": 0.1734408298186842, + "roc-auc": 0.9583120395781356, + "score": 0.9583120395781356 + }, + "test": { + "0": { + "precision": 0.9324796274738067, + "recall": 0.9748231535711569, + "f1-score": 0.9531813617939088, + "support": 13147.0 + }, + "1": { + "precision": 0.8702469619756958, + "recall": 0.7052096569250318, + "f1-score": 0.779084049833304, + "support": 3148.0 + }, + "accuracy": 0.922737035900583, + "macro avg": { + "precision": 0.9013632947247513, + "recall": 0.8400164052480943, + "f1-score": 0.8661327058136064, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9204570174101029, + "recall": 0.922737035900583, + "f1-score": 0.9195478338373586, + "support": 16295.0 + }, + "cross-entropy": 0.17694458755574416, + "roc-auc": 0.9627483366901237, + "score": 0.9627483366901237 + } + }, + "time": "0:05:03.738226", + "diversity": { + "train": { + "mean_centered_corr": 0.049995824694633484, + "mean_pairwise_disagreement": 0.03397909919872533, + "member_std": 0.031108465045690536 + }, + "val": { + "mean_centered_corr": 0.05969587340950966, + "mean_pairwise_disagreement": 0.03266793800197989, + "member_std": 0.03460051864385605 + }, + "test": { + "mean_centered_corr": 0.07376471161842346, + "mean_pairwise_disagreement": 0.04182474833958566, + "member_std": 0.03703681379556656 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..06391c184 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,290 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9495084683651196, + "recall": 0.9824752295695468, + "f1-score": 0.9657105817533626, + "support": 181971.0 + }, + "1": { + "precision": 0.9114928811301379, + "recall": 0.7755082764646155, + "f1-score": 0.8380199030364889, + "support": 42349.0 + }, + "accuracy": 0.9434022824536377, + "macro avg": { + "precision": 0.9305006747476288, + "recall": 0.8789917530170812, + "f1-score": 0.9018652423949257, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9423315688295711, + "recall": 0.9434022824536377, + "f1-score": 0.9416040707290184, + "support": 224320.0 + }, + "cross-entropy": 0.13410595787789584, + "roc-auc": 0.9804656245919036, + "score": 0.9804656245919036 + }, + "val": { + "0": { + "precision": 0.9362795477903392, + "recall": 0.9795113792485515, + "f1-score": 0.957407677711283, + "support": 16741.0 + }, + "1": { + "precision": 0.8692835365853658, + "recall": 0.6714748307329997, + "f1-score": 0.7576814482644079, + "support": 3397.0 + }, + "accuracy": 0.927549905651008, + "macro avg": { + "precision": 0.9027815421878524, + "recall": 0.8254931049907757, + "f1-score": 0.8575445629878454, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9249782542128591, + "recall": 0.927549905651008, + "f1-score": 0.9237166457601939, + "support": 20138.0 + }, + "cross-entropy": 0.17278545942960094, + "roc-auc": 0.958991133281215, + "score": 0.958991133281215 + }, + "test": { + "0": { + "precision": 0.9319371727748691, + "recall": 0.9748231535711569, + "f1-score": 0.9528978772445073, + "support": 13147.0 + }, + "1": { + "precision": 0.8698387731026347, + "recall": 0.7026683608640406, + "f1-score": 0.7773677736777368, + "support": 3148.0 + }, + "accuracy": 0.9222460877569807, + "macro avg": { + "precision": 0.9008879729387519, + "recall": 0.8387457572175987, + "f1-score": 0.865132825461122, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9199405012702239, + "recall": 0.9222460877569807, + "f1-score": 0.9189875510077357, + "support": 16295.0 + }, + "cross-entropy": 0.1769733127255982, + "roc-auc": 0.9627938464179217, + "score": 0.9627938464179217 + } + }, + "time": "0:05:03.738226", + "diversity": { + "train": { + "mean_centered_corr": 0.049995824694633484, + "mean_pairwise_disagreement": 0.03397909919872533, + "member_std": 0.031108465045690536 + }, + "val": { + "mean_centered_corr": 0.05969587340950966, + "mean_pairwise_disagreement": 0.03266793800197989, + "member_std": 0.03460051864385605 + }, + "test": { + "mean_centered_corr": 0.07376471161842346, + "mean_pairwise_disagreement": 0.04182474833958566, + "member_std": 0.03703681379556656 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 17, + 23, + 5, + 26, + 21, + 24, + 30, + 9, + 28, + 15, + 22, + 12, + 20, + 29, + 23, + 26, + 5, + 21, + 30 + ], + "head_selection_time": "0:00:17.296804" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..69dc71c4f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-best-head-evaluation/0/report.json @@ -0,0 +1,272 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9467367680900025, + "recall": 0.976688593237384, + "f1-score": 0.9614794738422339, + "support": 181971.0 + }, + "1": { + "precision": 0.8840730214254482, + "recall": 0.7638905287019764, + "f1-score": 0.8195994476887802, + "support": 42349.0 + }, + "accuracy": 0.9365148002853068, + "macro avg": { + "precision": 0.9154048947577254, + "recall": 0.8702895609696801, + "f1-score": 0.890539460765507, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9349065834988058, + "recall": 0.9365148002853068, + "f1-score": 0.9346941794967782, + "support": 224320.0 + }, + "cross-entropy": 0.14560289443931593, + "roc-auc": 0.9752118908061648, + "score": 0.9752118908061648 + }, + "val": { + "0": { + "precision": 0.9361300291878899, + "recall": 0.9770623021324891, + "f1-score": 0.9561582977728416, + "support": 16741.0 + }, + "1": { + "precision": 0.8559099437148218, + "recall": 0.6714748307329997, + "f1-score": 0.7525569119102606, + "support": 3397.0 + }, + "accuracy": 0.9255139537193365, + "macro avg": { + "precision": 0.8960199864513558, + "recall": 0.8242685664327445, + "f1-score": 0.8543576048415511, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9225980185437339, + "recall": 0.9255139537193365, + "f1-score": 0.9218135809302958, + "support": 20138.0 + }, + "cross-entropy": 0.18142101466285146, + "roc-auc": 0.9554589123032324, + "score": 0.9554589123032324 + }, + "test": { + "0": { + "precision": 0.9283744557329463, + "recall": 0.973073705027763, + "f1-score": 0.9501986853344226, + "support": 13147.0 + }, + "1": { + "precision": 0.8592445328031809, + "recall": 0.6864675984752223, + "f1-score": 0.7631997174642415, + "support": 3148.0 + }, + "accuracy": 0.9177048174286591, + "macro avg": { + "precision": 0.8938094942680637, + "recall": 0.8297706517514927, + "f1-score": 0.8566992013993321, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9150193776486933, + "recall": 0.9177048174286591, + "f1-score": 0.91407271105671, + "support": 16295.0 + }, + "cross-entropy": 0.18737908577750867, + "roc-auc": 0.9585579261153012, + "score": 0.9585579261153012 + } + }, + "time": "0:05:01.917656", + "diversity": { + "train": { + "mean_centered_corr": 0.049148593097925186, + "mean_pairwise_disagreement": 0.03323895011734389, + "member_std": 0.030483398586511612 + }, + "val": { + "mean_centered_corr": 0.056430645287036896, + "mean_pairwise_disagreement": 0.03106348453733753, + "member_std": 0.03352737799286842 + }, + "test": { + "mean_centered_corr": 0.07052021473646164, + "mean_pairwise_disagreement": 0.039438181116313135, + "member_std": 0.03553345799446106 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 26 + ], + "head_selection_time": "0:00:00.710724" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0.toml new file mode 100644 index 000000000..a6e4b28be --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.01 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0/report.json new file mode 100644 index 000000000..73f544f35 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/0/report.json @@ -0,0 +1,268 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9491977758441393, + "recall": 0.9831291799242736, + "f1-score": 0.9658655624151363, + "support": 181971.0 + }, + "1": { + "precision": 0.9143510768887401, + "recall": 0.7739025714892913, + "f1-score": 0.8382847569475529, + "support": 42349.0 + }, + "accuracy": 0.9436296362339515, + "macro avg": { + "precision": 0.9317744263664397, + "recall": 0.8785158757067825, + "f1-score": 0.9020751596813446, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9426191254604812, + "recall": 0.9436296362339515, + "f1-score": 0.9417797941789261, + "support": 224320.0 + }, + "cross-entropy": 0.13432585053967652, + "roc-auc": 0.9806862893380656, + "score": 0.9806862893380656 + }, + "val": { + "0": { + "precision": 0.936262482168331, + "recall": 0.9801087151305179, + "f1-score": 0.957684001634273, + "support": 16741.0 + }, + "1": { + "precision": 0.8725602755453502, + "recall": 0.6711804533411834, + "f1-score": 0.7587354409317804, + "support": 3397.0 + }, + "accuracy": 0.927996821928692, + "macro avg": { + "precision": 0.9044113788568406, + "recall": 0.8256445842358506, + "f1-score": 0.8582097212830266, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9255168075284329, + "recall": 0.927996821928692, + "f1-score": 0.9241241515644365, + "support": 20138.0 + }, + "cross-entropy": 0.17398384502840505, + "roc-auc": 0.9580370399944422, + "score": 0.9580370399944422 + }, + "test": { + "0": { + "precision": 0.9325081907535493, + "recall": 0.9742146497299764, + "f1-score": 0.9529052897849862, + "support": 13147.0 + }, + "1": { + "precision": 0.867578125, + "recall": 0.7055273189326556, + "f1-score": 0.7782060266292922, + "support": 3148.0 + }, + "accuracy": 0.9223074562749309, + "macro avg": { + "precision": 0.9000431578767747, + "recall": 0.839870984331316, + "f1-score": 0.8655556582071392, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9199644750743733, + "recall": 0.9223074562749309, + "f1-score": 0.9191554720240703, + "support": 16295.0 + }, + "cross-entropy": 0.17817953563365546, + "roc-auc": 0.9621106810110944, + "score": 0.9621106810110944 + } + }, + "time": "0:05:01.917656", + "diversity": { + "train": { + "mean_centered_corr": 0.049148593097925186, + "mean_pairwise_disagreement": 0.03323895011734389, + "member_std": 0.030483398586511612 + }, + "val": { + "mean_centered_corr": 0.056430645287036896, + "mean_pairwise_disagreement": 0.03106348453733753, + "member_std": 0.03352737799286842 + }, + "test": { + "mean_centered_corr": 0.07052021473646164, + "mean_pairwise_disagreement": 0.039438181116313135, + "member_std": 0.03553345799446106 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..999ba0393 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,282 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9505159911985807, + "recall": 0.9804254524072517, + "f1-score": 0.9652390793901555, + "support": 181971.0 + }, + "1": { + "precision": 0.9027387161073642, + "recall": 0.7806795910174975, + "f1-score": 0.8372841006939168, + "support": 42349.0 + }, + "accuracy": 0.942715763195435, + "macro avg": { + "precision": 0.9266273536529724, + "recall": 0.8805525217123746, + "f1-score": 0.9012615900420362, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9414961988357154, + "recall": 0.942715763195435, + "f1-score": 0.9410826716119502, + "support": 224320.0 + }, + "cross-entropy": 0.1373233192307072, + "roc-auc": 0.9795565058577262, + "score": 0.9795565058577262 + }, + "val": { + "0": { + "precision": 0.9383593033913841, + "recall": 0.9784361746610118, + "f1-score": 0.9579787700675498, + "support": 16741.0 + }, + "1": { + "precision": 0.8653989560029829, + "recall": 0.6832499264056521, + "f1-score": 0.7636124362559632, + "support": 3397.0 + }, + "accuracy": 0.9286423676631245, + "macro avg": { + "precision": 0.9018791296971835, + "recall": 0.830843050533332, + "f1-score": 0.8607956031617565, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9260519094059635, + "recall": 0.9286423676631245, + "f1-score": 0.9251918778261178, + "support": 20138.0 + }, + "cross-entropy": 0.17333139914835735, + "roc-auc": 0.9587733087820138, + "score": 0.9587733087820138 + }, + "test": { + "0": { + "precision": 0.9348479296445584, + "recall": 0.9702593747623032, + "f1-score": 0.9522245446401911, + "support": 13147.0 + }, + "1": { + "precision": 0.8524528301886792, + "recall": 0.7175984752223634, + "f1-score": 0.7792342186961021, + "support": 3148.0 + }, + "accuracy": 0.9214482970236268, + "macro avg": { + "precision": 0.8936503799166189, + "recall": 0.8439289249923333, + "f1-score": 0.8657293816681466, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9189301773839197, + "recall": 0.9214482970236268, + "f1-score": 0.9188048732028181, + "support": 16295.0 + }, + "cross-entropy": 0.17776343990066415, + "roc-auc": 0.9626104060922291, + "score": 0.9626104060922291 + } + }, + "time": "0:05:01.917656", + "diversity": { + "train": { + "mean_centered_corr": 0.049148593097925186, + "mean_pairwise_disagreement": 0.03323895011734389, + "member_std": 0.030483398586511612 + }, + "val": { + "mean_centered_corr": 0.056430645287036896, + "mean_pairwise_disagreement": 0.03106348453733753, + "member_std": 0.03352737799286842 + }, + "test": { + "mean_centered_corr": 0.07052021473646164, + "mean_pairwise_disagreement": 0.039438181116313135, + "member_std": 0.03553345799446106 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 26, + 30, + 24, + 5, + 17, + 31, + 22, + 13, + 15, + 20, + 24 + ], + "head_selection_time": "0:00:11.234254" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4003e9150 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,657 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9432566752157673, + "recall": 0.9741662132977232, + "f1-score": 0.9584623078898636, + "support": 181971.0 + }, + "1": { + "precision": 0.8708019567965701, + "recall": 0.7481876785756453, + "f1-score": 0.8048517177875151, + "support": 42349.0 + }, + "accuracy": 0.9315041012838802, + "macro avg": { + "precision": 0.9070293160061687, + "recall": 0.8611769459366843, + "f1-score": 0.8816570128386894, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9295780693387408, + "recall": 0.9315041012838802, + "f1-score": 0.9294624198716558, + "support": 224320.0 + }, + "cross-entropy": 0.15818911070182506, + "roc-auc": 0.9710347146934778, + "score": 0.9710347146934778 + }, + "val": { + "0": { + "precision": 0.9363334478415206, + "recall": 0.9768831013678991, + "f1-score": 0.9561785599438711, + "support": 16741.0 + }, + "1": { + "precision": 0.8551646706586826, + "recall": 0.672652340300265, + "f1-score": 0.753007085187016, + "support": 3397.0 + }, + "accuracy": 0.9255636110835237, + "macro avg": { + "precision": 0.8957490592501016, + "recall": 0.824767720834082, + "f1-score": 0.8545928225654436, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9226414061248607, + "recall": 0.9255636110835237, + "f1-score": 0.9219063631145417, + "support": 20138.0 + }, + "cross-entropy": 0.1787492077720551, + "roc-auc": 0.9576967765860231, + "score": 0.9576967765860231 + }, + "test": { + "0": { + "precision": 0.9309967939376275, + "recall": 0.971856697345402, + "f1-score": 0.9509880540359495, + "support": 13147.0 + }, + "1": { + "precision": 0.8560871256320498, + "recall": 0.6991740787801779, + "f1-score": 0.7697149851372618, + "support": 3148.0 + }, + "accuracy": 0.9191776618594661, + "macro avg": { + "precision": 0.8935419597848386, + "recall": 0.8355153880627899, + "f1-score": 0.8603515195866056, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9165251378574827, + "recall": 0.9191776618594661, + "f1-score": 0.915968255269882, + "support": 16295.0 + }, + "cross-entropy": 0.18291214189215663, + "roc-auc": 0.9617315500639867, + "score": 0.9617315500639867 + } + }, + "time": "0:03:52.600341", + "diversity": { + "train": { + "mean_centered_corr": 0.0775546282529831, + "mean_pairwise_disagreement": 0.0356185432101606, + "member_std": 0.03550577908754349 + }, + "val": { + "mean_centered_corr": 0.08212897181510925, + "mean_pairwise_disagreement": 0.03171593825186856, + "member_std": 0.03616226091980934 + }, + "test": { + "mean_centered_corr": 0.10865671932697296, + "mean_pairwise_disagreement": 0.038570855892862445, + "member_std": 0.038954634219408035 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:01.289307" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..10f998b1d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0.toml @@ -0,0 +1,58 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..4fb8df0ec --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0/report.json @@ -0,0 +1,653 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9398710962062461, + "recall": 0.9856790367695951, + "f1-score": 0.9622301917057169, + "support": 181971.0 + }, + "1": { + "precision": 0.9221624850657109, + "recall": 0.7290372854140593, + "f1-score": 0.8143058724234792, + "support": 42349.0 + }, + "accuracy": 0.9372280670470756, + "macro avg": { + "precision": 0.9310167906359785, + "recall": 0.8573581610918273, + "f1-score": 0.888268032064598, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.93652791693917, + "recall": 0.9372280670470756, + "f1-score": 0.9343038053055588, + "support": 224320.0 + }, + "cross-entropy": 0.14711870269948546, + "roc-auc": 0.9762437618004894, + "score": 0.9762437618004894 + }, + "val": { + "0": { + "precision": 0.9303347987825499, + "recall": 0.9859626067737889, + "f1-score": 0.9573412986109098, + "support": 16741.0 + }, + "1": { + "precision": 0.9019198664440734, + "recall": 0.6361495437150427, + "f1-score": 0.7460728465389262, + "support": 3397.0 + }, + "accuracy": 0.9269540172807628, + "macro avg": { + "precision": 0.9161273326133117, + "recall": 0.8110560752444158, + "f1-score": 0.8517070725749181, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9255415956266354, + "recall": 0.9269540172807628, + "f1-score": 0.9217032545306373, + "support": 20138.0 + }, + "cross-entropy": 0.175030748761162, + "roc-auc": 0.9589238560635404, + "score": 0.9589238560635404 + }, + "test": { + "0": { + "precision": 0.925214899713467, + "recall": 0.9824294515859131, + "f1-score": 0.9529641789943557, + "support": 13147.0 + }, + "1": { + "precision": 0.9010706638115632, + "recall": 0.6683608640406608, + "f1-score": 0.7674630676636878, + "support": 3148.0 + }, + "accuracy": 0.9217551396133783, + "macro avg": { + "precision": 0.9131427817625151, + "recall": 0.825395157813287, + "f1-score": 0.8602136233290217, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9205505207862382, + "recall": 0.9217551396133783, + "f1-score": 0.9171275727673571, + "support": 16295.0 + }, + "cross-entropy": 0.17966720795996355, + "roc-auc": 0.9624954901031625, + "score": 0.9624954901031625 + } + }, + "time": "0:03:52.600341", + "diversity": { + "train": { + "mean_centered_corr": 0.0775546282529831, + "mean_pairwise_disagreement": 0.0356185432101606, + "member_std": 0.03550577908754349 + }, + "val": { + "mean_centered_corr": 0.08212897181510925, + "mean_pairwise_disagreement": 0.03171593825186856, + "member_std": 0.03616226091980934 + }, + "test": { + "mean_centered_corr": 0.10865671932697296, + "mean_pairwise_disagreement": 0.038570855892862445, + "member_std": 0.038954634219408035 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..fb5104edb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,666 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9379184990964076, + "recall": 0.986816580663952, + "f1-score": 0.9617464096253906, + "support": 181971.0 + }, + "1": { + "precision": 0.9269977481589678, + "recall": 0.7193322156367329, + "f1-score": 0.8100676762707583, + "support": 42349.0 + }, + "accuracy": 0.9363186519258203, + "macro avg": { + "precision": 0.9324581236276877, + "recall": 0.8530743981503424, + "f1-score": 0.8859070429480744, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9358567886762504, + "recall": 0.9363186519258203, + "f1-score": 0.9331112336320092, + "support": 224320.0 + }, + "cross-entropy": 0.14950830177513505, + "roc-auc": 0.9756752367295682, + "score": 0.9756752367295682 + }, + "val": { + "0": { + "precision": 0.9291661978170361, + "recall": 0.9865002090675586, + "f1-score": 0.9569752281616688, + "support": 16741.0 + }, + "1": { + "precision": 0.9043993231810491, + "recall": 0.6293788637032676, + "f1-score": 0.7422322513452526, + "support": 3397.0 + }, + "accuracy": 0.9262588141821432, + "macro avg": { + "precision": 0.9167827604990426, + "recall": 0.8079395363854132, + "f1-score": 0.8496037397534607, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9249883711640194, + "recall": 0.9262588141821432, + "f1-score": 0.9207510801705393, + "support": 20138.0 + }, + "cross-entropy": 0.1746148873886249, + "roc-auc": 0.9597570226838343, + "score": 0.9597570226838343 + }, + "test": { + "0": { + "precision": 0.9218071860547847, + "recall": 0.9854719707918156, + "f1-score": 0.9525770163958532, + "support": 13147.0 + }, + "1": { + "precision": 0.9147321428571429, + "recall": 0.6508894536213469, + "f1-score": 0.7605790645879733, + "support": 3148.0 + }, + "accuracy": 0.920834611844124, + "macro avg": { + "precision": 0.9182696644559638, + "recall": 0.8181807122065812, + "f1-score": 0.8565780404919132, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9204403719408739, + "recall": 0.920834611844124, + "f1-score": 0.9154852979367427, + "support": 16295.0 + }, + "cross-entropy": 0.17973653945792817, + "roc-auc": 0.9633062567165207, + "score": 0.9633062567165207 + } + }, + "time": "0:03:52.600341", + "diversity": { + "train": { + "mean_centered_corr": 0.0775546282529831, + "mean_pairwise_disagreement": 0.0356185432101606, + "member_std": 0.03550577908754349 + }, + "val": { + "mean_centered_corr": 0.08212897181510925, + "mean_pairwise_disagreement": 0.03171593825186856, + "member_std": 0.03616226091980934 + }, + "test": { + "mean_centered_corr": 0.10865671932697296, + "mean_pairwise_disagreement": 0.038570855892862445, + "member_std": 0.038954634219408035 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 28, + 30, + 17, + 22, + 12, + 29, + 28, + 18, + 6 + ], + "head_selection_time": "0:00:13.698772" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ba4f4903d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-best-head-evaluation/0/report.json @@ -0,0 +1,298 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7051174640655518, + "min_features_per_member": 194.0, + "mean_features_per_member": 210.125, + "max_features_per_member": 226.0, + "min_members_per_feature": 15.0, + "mean_members_per_feature": 22.563758850097656, + "max_members_per_feature": 30.0, + "mean_pairwise_jaccard": 0.5455597077325217, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9507069533217122, + "recall": 0.971066818339186, + "f1-score": 0.9607790364806342, + "support": 181971.0 + }, + "1": { + "precision": 0.8630760428586289, + "recall": 0.7836548678835391, + "f1-score": 0.8214502295763666, + "support": 42349.0 + }, + "accuracy": 0.9356856276747504, + "macro avg": { + "precision": 0.9068914980901706, + "recall": 0.8773608431113625, + "f1-score": 0.8911146330285005, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9341632593702094, + "recall": 0.9356856276747504, + "f1-score": 0.9344753825773316, + "support": 224320.0 + }, + "cross-entropy": 0.14753096113026493, + "roc-auc": 0.975139322227928, + "score": 0.975139322227928 + }, + "val": { + "0": { + "precision": 0.9388616006934412, + "recall": 0.9704916074308584, + "f1-score": 0.9544146155201786, + "support": 16741.0 + }, + "1": { + "precision": 0.8256265442993294, + "recall": 0.6885487194583456, + "f1-score": 0.7508828250401284, + "support": 3397.0 + }, + "accuracy": 0.9229317707816069, + "macro avg": { + "precision": 0.8822440724963853, + "recall": 0.829520163444602, + "f1-score": 0.8526487202801535, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9197604244807687, + "recall": 0.9229317707816069, + "f1-score": 0.9200816384489338, + "support": 20138.0 + }, + "cross-entropy": 0.18416591007766794, + "roc-auc": 0.9540607823461205, + "score": 0.9540607823461205 + }, + "test": { + "0": { + "precision": 0.9347538574577516, + "recall": 0.9676732334372861, + "f1-score": 0.9509287289307471, + "support": 13147.0 + }, + "1": { + "precision": 0.8417132216014898, + "recall": 0.7179161372299873, + "f1-score": 0.7749014229384537, + "support": 3148.0 + }, + "accuracy": 0.9194231359312672, + "macro avg": { + "precision": 0.8882335395296207, + "recall": 0.8427946853336367, + "f1-score": 0.8629150759346004, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9167795143049124, + "recall": 0.9194231359312672, + "f1-score": 0.9169223491048043, + "support": 16295.0 + }, + "cross-entropy": 0.1854429639680667, + "roc-auc": 0.9604495578247303, + "score": 0.9604495578247303 + } + }, + "time": "0:03:07.457038", + "diversity": { + "train": { + "mean_centered_corr": 0.1248728483915329, + "mean_pairwise_disagreement": 0.057014991184828126, + "member_std": 0.05655066296458244 + }, + "val": { + "mean_centered_corr": 0.11596227437257767, + "mean_pairwise_disagreement": 0.055349840135324326, + "member_std": 0.06094019487500191 + }, + "test": { + "mean_centered_corr": 0.14900994300842285, + "mean_pairwise_disagreement": 0.06749510041671203, + "member_std": 0.06686553359031677 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 20 + ], + "head_selection_time": "0:00:01.696307" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0.toml new file mode 100644 index 000000000..137e9f011 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.7 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0/report.json new file mode 100644 index 000000000..c760f4271 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-evaluation/0/report.json @@ -0,0 +1,294 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7051174640655518, + "min_features_per_member": 194.0, + "mean_features_per_member": 210.125, + "max_features_per_member": 226.0, + "min_members_per_feature": 15.0, + "mean_members_per_feature": 22.563758850097656, + "max_members_per_feature": 30.0, + "mean_pairwise_jaccard": 0.5455597077325217, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9454041901191536, + "recall": 0.9867176638035731, + "f1-score": 0.9656192355886355, + "support": 181971.0 + }, + "1": { + "precision": 0.9297322440910545, + "recall": 0.7551536045715365, + "f1-score": 0.8333984833085764, + "support": 42349.0 + }, + "accuracy": 0.9430010699001427, + "macro avg": { + "precision": 0.9375682171051041, + "recall": 0.8709356341875547, + "f1-score": 0.8995088594486059, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9424455094738968, + "recall": 0.9430010699001427, + "f1-score": 0.9406574995048791, + "support": 224320.0 + }, + "cross-entropy": 0.1482790588601011, + "roc-auc": 0.9807497384825534, + "score": 0.9807497384825534 + }, + "val": { + "0": { + "precision": 0.9299046974567191, + "recall": 0.9850068693626426, + "f1-score": 0.9566629924000696, + "support": 16741.0 + }, + "1": { + "precision": 0.8956340956340957, + "recall": 0.6340889019723285, + "f1-score": 0.7425025853154085, + "support": 3397.0 + }, + "accuracy": 0.9258118979044593, + "macro avg": { + "precision": 0.9127693965454073, + "recall": 0.8095478856674856, + "f1-score": 0.849582788857739, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9241237244508868, + "recall": 0.9258118979044593, + "f1-score": 0.9205371158052441, + "support": 20138.0 + }, + "cross-entropy": 0.1836966792126167, + "roc-auc": 0.9574455596570353, + "score": 0.9574455596570353 + }, + "test": { + "0": { + "precision": 0.9292490404808458, + "recall": 0.9760401612535179, + "f1-score": 0.9520700400652916, + "support": 13147.0 + }, + "1": { + "precision": 0.8732904263877715, + "recall": 0.6896442185514613, + "f1-score": 0.7706780262690806, + "support": 3148.0 + }, + "accuracy": 0.9207118748082234, + "macro avg": { + "precision": 0.9012697334343087, + "recall": 0.8328421899024896, + "f1-score": 0.8613740331671861, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9184385024529232, + "recall": 0.9207118748082234, + "f1-score": 0.9170272625611203, + "support": 16295.0 + }, + "cross-entropy": 0.1892419229433621, + "roc-auc": 0.9607387445394369, + "score": 0.9607387445394369 + } + }, + "time": "0:03:07.457038", + "diversity": { + "train": { + "mean_centered_corr": 0.1248728483915329, + "mean_pairwise_disagreement": 0.057014991184828126, + "member_std": 0.05655066296458244 + }, + "val": { + "mean_centered_corr": 0.11596227437257767, + "mean_pairwise_disagreement": 0.055349840135324326, + "member_std": 0.06094019487500191 + }, + "test": { + "mean_centered_corr": 0.14900994300842285, + "mean_pairwise_disagreement": 0.06749510041671203, + "member_std": 0.06686553359031677 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..476c62b54 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.7-greedy-heads-evaluation/0/report.json @@ -0,0 +1,305 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7051174640655518, + "min_features_per_member": 194.0, + "mean_features_per_member": 210.125, + "max_features_per_member": 226.0, + "min_members_per_feature": 15.0, + "mean_members_per_feature": 22.563758850097656, + "max_members_per_feature": 30.0, + "mean_pairwise_jaccard": 0.5455597077325217, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9461931452336468, + "recall": 0.9852064339922295, + "f1-score": 0.9653057653219185, + "support": 181971.0 + }, + "1": { + "precision": 0.9227457957871779, + "recall": 0.7592623202436893, + "f1-score": 0.8330591359544012, + "support": 42349.0 + }, + "accuracy": 0.9425508202567761, + "macro avg": { + "precision": 0.9344694705104124, + "recall": 0.8722343771179595, + "f1-score": 0.8991824506381598, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9417665590990689, + "recall": 0.9425508202567761, + "f1-score": 0.9403391439458264, + "support": 224320.0 + }, + "cross-entropy": 0.14359261649483174, + "roc-auc": 0.9800293959977575, + "score": 0.9800293959977575 + }, + "val": { + "0": { + "precision": 0.9331481061465184, + "recall": 0.9830356609521534, + "f1-score": 0.9574424760741194, + "support": 16741.0 + }, + "1": { + "precision": 0.8864908073541167, + "recall": 0.6529290550485722, + "f1-score": 0.7519918630276318, + "support": 3397.0 + }, + "accuracy": 0.9273512761942596, + "macro avg": { + "precision": 0.9098194567503175, + "recall": 0.8179823580003629, + "f1-score": 0.8547171695508755, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9252776699563413, + "recall": 0.9273512761942596, + "f1-score": 0.9227858203725146, + "support": 20138.0 + }, + "cross-entropy": 0.17676896401301107, + "roc-auc": 0.9588480241238587, + "score": 0.9588480241238587 + }, + "test": { + "0": { + "precision": 0.929438982070561, + "recall": 0.9778656727770594, + "f1-score": 0.9530375477223025, + "support": 13147.0 + }, + "1": { + "precision": 0.881851400730816, + "recall": 0.6899618805590851, + "f1-score": 0.7741935483870968, + "support": 3148.0 + }, + "accuracy": 0.9222460877569807, + "macro avg": { + "precision": 0.9056451914006884, + "recall": 0.8339137766680722, + "f1-score": 0.8636155480546996, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9202456279093141, + "recall": 0.9222460877569807, + "f1-score": 0.9184870162766303, + "support": 16295.0 + }, + "cross-entropy": 0.18025747533530698, + "roc-auc": 0.9630162122394903, + "score": 0.9630162122394903 + } + }, + "time": "0:03:07.457038", + "diversity": { + "train": { + "mean_centered_corr": 0.1248728483915329, + "mean_pairwise_disagreement": 0.057014991184828126, + "member_std": 0.05655066296458244 + }, + "val": { + "mean_centered_corr": 0.11596227437257767, + "mean_pairwise_disagreement": 0.055349840135324326, + "member_std": 0.06094019487500191 + }, + "test": { + "mean_centered_corr": 0.14900994300842285, + "mean_pairwise_disagreement": 0.06749510041671203, + "member_std": 0.06686553359031677 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 20, + 28, + 22, + 4, + 16, + 13, + 10, + 26 + ], + "head_selection_time": "0:00:10.918451" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e7eac6ed6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-best-head-evaluation/0/report.json @@ -0,0 +1,298 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.805578887462616, + "min_features_per_member": 224.0, + "mean_features_per_member": 240.0625, + "max_features_per_member": 251.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.77852439880371, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6740198042716817, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9487459340304847, + "recall": 0.9697259453429392, + "f1-score": 0.9591212231564873, + "support": 181971.0 + }, + "1": { + "precision": 0.8562557077625571, + "recall": 0.7748943304446386, + "f1-score": 0.8135458759947443, + "support": 42349.0 + }, + "accuracy": 0.9329440085592011, + "macro avg": { + "precision": 0.9025008208965208, + "recall": 0.872310137893789, + "f1-score": 0.8863335495756157, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9312848579239384, + "recall": 0.9329440085592011, + "f1-score": 0.9316382952991734, + "support": 224320.0 + }, + "cross-entropy": 0.15551409232715008, + "roc-auc": 0.9725736580353728, + "score": 0.9725736580353728 + }, + "val": { + "0": { + "precision": 0.9397257116514924, + "recall": 0.9741353563108536, + "f1-score": 0.9566212054553453, + "support": 16741.0 + }, + "1": { + "precision": 0.8444683908045977, + "recall": 0.6920812481601413, + "f1-score": 0.7607183303672544, + "support": 3397.0 + }, + "accuracy": 0.9265567583672659, + "macro avg": { + "precision": 0.8920970512280451, + "recall": 0.8331083022354975, + "f1-score": 0.8586697679112999, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9236571289264501, + "recall": 0.9265567583672659, + "f1-score": 0.9235751201105123, + "support": 20138.0 + }, + "cross-entropy": 0.18078572473789445, + "roc-auc": 0.9557633918282306, + "score": 0.9557633918282306 + }, + "test": { + "0": { + "precision": 0.9362878843743087, + "recall": 0.9657716589335971, + "f1-score": 0.9508012580500225, + "support": 13147.0 + }, + "1": { + "precision": 0.8354059985369422, + "recall": 0.7255400254129606, + "f1-score": 0.7766065963957838, + "support": 3148.0 + }, + "accuracy": 0.919361767413317, + "macro avg": { + "precision": 0.8858469414556254, + "recall": 0.8456558421732788, + "f1-score": 0.8637039272229031, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9167987050790629, + "recall": 0.919361767413317, + "f1-score": 0.9171489232916583, + "support": 16295.0 + }, + "cross-entropy": 0.1855701112620712, + "roc-auc": 0.9594055402651032, + "score": 0.9594055402651032 + } + }, + "time": "0:03:00.284704", + "diversity": { + "train": { + "mean_centered_corr": 0.10049896687269211, + "mean_pairwise_disagreement": 0.04618599113881092, + "member_std": 0.04431634396314621 + }, + "val": { + "mean_centered_corr": 0.10512824356555939, + "mean_pairwise_disagreement": 0.042368444026539456, + "member_std": 0.046922747045755386 + }, + "test": { + "mean_centered_corr": 0.1312195509672165, + "mean_pairwise_disagreement": 0.05293071791267854, + "member_std": 0.051259808242321014 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 17 + ], + "head_selection_time": "0:00:01.208354" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0.toml new file mode 100644 index 000000000..1173eceb6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.8 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0/report.json new file mode 100644 index 000000000..3d703a52e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-evaluation/0/report.json @@ -0,0 +1,294 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.805578887462616, + "min_features_per_member": 224.0, + "mean_features_per_member": 240.0625, + "max_features_per_member": 251.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.77852439880371, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6740198042716817, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.945464060302995, + "recall": 0.9829093646789873, + "f1-score": 0.963823154582122, + "support": 181971.0 + }, + "1": { + "precision": 0.9115019065505663, + "recall": 0.7563814966114902, + "f1-score": 0.8267282652178962, + "support": 42349.0 + }, + "accuracy": 0.940143544935806, + "macro avg": { + "precision": 0.9284829834267807, + "recall": 0.8696454306452388, + "f1-score": 0.895275709900009, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9390524017381697, + "recall": 0.940143544935806, + "f1-score": 0.9379412382586306, + "support": 224320.0 + }, + "cross-entropy": 0.14707160080340106, + "roc-auc": 0.9781088321813957, + "score": 0.9781088321813957 + }, + "val": { + "0": { + "precision": 0.9330230446134635, + "recall": 0.9819007227764172, + "f1-score": 0.9568380919118717, + "support": 16741.0 + }, + "1": { + "precision": 0.8797619047619047, + "recall": 0.652634677656756, + "f1-score": 0.749366232888288, + "support": 3397.0 + }, + "accuracy": 0.9263581289105174, + "macro avg": { + "precision": 0.9063924746876841, + "recall": 0.8172677002165866, + "f1-score": 0.8531021624000799, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9240386324535794, + "recall": 0.9263581289105174, + "f1-score": 0.9218404801776819, + "support": 20138.0 + }, + "cross-entropy": 0.17738371149581372, + "roc-auc": 0.9580661506671708, + "score": 0.9580661506671708 + }, + "test": { + "0": { + "precision": 0.930227503260397, + "recall": 0.9765726021145509, + "f1-score": 0.9528368399569558, + "support": 13147.0 + }, + "1": { + "precision": 0.8764540713999198, + "recall": 0.6940914866581956, + "f1-score": 0.7746853394788158, + "support": 3148.0 + }, + "accuracy": 0.9220006136851795, + "macro avg": { + "precision": 0.9033407873301584, + "recall": 0.8353320443863732, + "f1-score": 0.8637610897178858, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9198391164241416, + "recall": 0.9220006136851795, + "f1-score": 0.9184200910459288, + "support": 16295.0 + }, + "cross-entropy": 0.18143273966935003, + "roc-auc": 0.9619981087669689, + "score": 0.9619981087669689 + } + }, + "time": "0:03:00.284704", + "diversity": { + "train": { + "mean_centered_corr": 0.10049896687269211, + "mean_pairwise_disagreement": 0.04618599113881092, + "member_std": 0.04431634396314621 + }, + "val": { + "mean_centered_corr": 0.10512824356555939, + "mean_pairwise_disagreement": 0.042368444026539456, + "member_std": 0.046922747045755386 + }, + "test": { + "mean_centered_corr": 0.1312195509672165, + "mean_pairwise_disagreement": 0.05293071791267854, + "member_std": 0.051259808242321014 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5c5e6a95f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.8-greedy-heads-evaluation/0/report.json @@ -0,0 +1,305 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.805578887462616, + "min_features_per_member": 224.0, + "mean_features_per_member": 240.0625, + "max_features_per_member": 251.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.77852439880371, + "max_members_per_feature": 31.0, + "mean_pairwise_jaccard": 0.6740198042716817, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9423305588585018, + "recall": 0.984250237675234, + "f1-score": 0.962834341745579, + "support": 181971.0 + }, + "1": { + "precision": 0.9163309394523267, + "recall": 0.7411745259628326, + "f1-score": 0.8194979308904351, + "support": 42349.0 + }, + "accuracy": 0.9383603780313837, + "macro avg": { + "precision": 0.9293307491554142, + "recall": 0.8627123818190333, + "f1-score": 0.891166136318007, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9374221339198779, + "recall": 0.9383603780313837, + "f1-score": 0.9357740989526738, + "support": 224320.0 + }, + "cross-entropy": 0.14563506125296627, + "roc-auc": 0.9772221952124672, + "score": 0.9772221952124672 + }, + "val": { + "0": { + "precision": 0.9319107991849672, + "recall": 0.9835135296577265, + "f1-score": 0.9570170594902496, + "support": 16741.0 + }, + "1": { + "precision": 0.8882591093117409, + "recall": 0.6458639976449808, + "f1-score": 0.7479120504516789, + "support": 3397.0 + }, + "accuracy": 0.9265567583672659, + "macro avg": { + "precision": 0.910084954248354, + "recall": 0.8146887636513537, + "f1-score": 0.8524645549709642, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.924547367339732, + "recall": 0.9265567583672659, + "f1-score": 0.9217439581046092, + "support": 20138.0 + }, + "cross-entropy": 0.17444583477097872, + "roc-auc": 0.9587700557017027, + "score": 0.9587700557017027 + }, + "test": { + "0": { + "precision": 0.9287004262081918, + "recall": 0.9778656727770594, + "f1-score": 0.9526491293071508, + "support": 13147.0 + }, + "1": { + "precision": 0.8813213703099511, + "recall": 0.6864675984752223, + "f1-score": 0.7717857142857143, + "support": 3148.0 + }, + "accuracy": 0.9215710340595274, + "macro avg": { + "precision": 0.9050108982590714, + "recall": 0.8321666356261408, + "f1-score": 0.8622174217964326, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9195473566796456, + "recall": 0.9215710340595274, + "f1-score": 0.9177084707930372, + "support": 16295.0 + }, + "cross-entropy": 0.179030199475604, + "roc-auc": 0.9623207965369405, + "score": 0.9623207965369405 + } + }, + "time": "0:03:00.284704", + "diversity": { + "train": { + "mean_centered_corr": 0.10049896687269211, + "mean_pairwise_disagreement": 0.04618599113881092, + "member_std": 0.04431634396314621 + }, + "val": { + "mean_centered_corr": 0.10512824356555939, + "mean_pairwise_disagreement": 0.042368444026539456, + "member_std": 0.046922747045755386 + }, + "test": { + "mean_centered_corr": 0.1312195509672165, + "mean_pairwise_disagreement": 0.05293071791267854, + "member_std": 0.051259808242321014 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 17, + 12, + 30, + 28, + 15, + 4, + 13, + 22 + ], + "head_selection_time": "0:00:13.417596" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-best-head-evaluation/0/report.json new file mode 100644 index 000000000..753101f67 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-best-head-evaluation/0/report.json @@ -0,0 +1,298 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9518098162802934, + "recall": 0.9742541393958378, + "f1-score": 0.9629012065730121, + "support": 181971.0 + }, + "1": { + "precision": 0.8768984182037942, + "recall": 0.7880469432572198, + "f1-score": 0.8301018568035121, + "support": 42349.0 + }, + "accuracy": 0.9391003922967189, + "macro avg": { + "precision": 0.9143541172420437, + "recall": 0.8811505413265288, + "f1-score": 0.8965015316882621, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9376674179335491, + "recall": 0.9391003922967189, + "f1-score": 0.9378302380308022, + "support": 224320.0 + }, + "cross-entropy": 0.1417006278725686, + "roc-auc": 0.9768342693042886, + "score": 0.9768342693042886 + }, + "val": { + "0": { + "precision": 0.9406342396660676, + "recall": 0.9691774684905322, + "f1-score": 0.9546925566343042, + "support": 16741.0 + }, + "1": { + "precision": 0.8213914849428868, + "recall": 0.6985575507801001, + "f1-score": 0.755011135857461, + "support": 3397.0 + }, + "accuracy": 0.9235276591518522, + "macro avg": { + "precision": 0.8810128623044773, + "recall": 0.8338675096353161, + "f1-score": 0.8548518462458826, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9205196484556869, + "recall": 0.9235276591518522, + "f1-score": 0.9210090832814918, + "support": 20138.0 + }, + "cross-entropy": 0.18275894278340477, + "roc-auc": 0.9560981250001208, + "score": 0.9560981250001208 + }, + "test": { + "0": { + "precision": 0.9386612939332886, + "recall": 0.9567962272761846, + "f1-score": 0.9476420069308422, + "support": 13147.0 + }, + "1": { + "precision": 0.8037318590186593, + "recall": 0.738881829733164, + "f1-score": 0.7699437272426349, + "support": 3148.0 + }, + "accuracy": 0.9146977600490949, + "macro avg": { + "precision": 0.8711965764759739, + "recall": 0.8478390285046743, + "f1-score": 0.8587928670867386, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9125945335091553, + "recall": 0.9146977600490949, + "f1-score": 0.9133128148806136, + "support": 16295.0 + }, + "cross-entropy": 0.1874554754465997, + "roc-auc": 0.9601610718172741, + "score": 0.9601610718172741 + } + }, + "time": "0:03:06.389407", + "diversity": { + "train": { + "mean_centered_corr": 0.08821944147348404, + "mean_pairwise_disagreement": 0.03961999131425153, + "member_std": 0.03912428021430969 + }, + "val": { + "mean_centered_corr": 0.09983313083648682, + "mean_pairwise_disagreement": 0.03728577252442021, + "member_std": 0.04294782876968384 + }, + "test": { + "mean_centered_corr": 0.11663155257701874, + "mean_pairwise_disagreement": 0.04569306832691603, + "member_std": 0.04552939534187317 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:01.042460" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0.toml new file mode 100644 index 000000000..fe5f765a4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0/report.json new file mode 100644 index 000000000..600b47205 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-evaluation/0/report.json @@ -0,0 +1,294 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.946151494708437, + "recall": 0.9865253254639476, + "f1-score": 0.9659167032097862, + "support": 181971.0 + }, + "1": { + "precision": 0.9291001619245894, + "recall": 0.7587428274575551, + "f1-score": 0.8353242431726308, + "support": 42349.0 + }, + "accuracy": 0.9435226462196862, + "macro avg": { + "precision": 0.9376258283165132, + "recall": 0.8726340764607513, + "f1-score": 0.9006204731912084, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9429324019299814, + "recall": 0.9435226462196862, + "f1-score": 0.94126236971249, + "support": 224320.0 + }, + "cross-entropy": 0.13654273925686128, + "roc-auc": 0.980952103177431, + "score": 0.980952103177431 + }, + "val": { + "0": { + "precision": 0.9326650809831238, + "recall": 0.9837524640105131, + "f1-score": 0.9575278351114858, + "support": 16741.0 + }, + "1": { + "precision": 0.8903225806451613, + "recall": 0.6499852811304092, + "f1-score": 0.7514037774374681, + "support": 3397.0 + }, + "accuracy": 0.9274505909226338, + "macro avg": { + "precision": 0.9114938308141426, + "recall": 0.8168688725704611, + "f1-score": 0.854465806274477, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9255224911704286, + "recall": 0.9274505909226338, + "f1-score": 0.9227575786848973, + "support": 20138.0 + }, + "cross-entropy": 0.17552008158175839, + "roc-auc": 0.9580942942079151, + "score": 0.9580942942079151 + }, + "test": { + "0": { + "precision": 0.9282410752998989, + "recall": 0.9770289799954363, + "f1-score": 0.9520103761348897, + "support": 13147.0 + }, + "1": { + "precision": 0.877085877085877, + "recall": 0.684561626429479, + "f1-score": 0.7689562890276539, + "support": 3148.0 + }, + "accuracy": 0.9205277692543725, + "macro avg": { + "precision": 0.902663476192888, + "recall": 0.8307953032124576, + "f1-score": 0.8604833325812717, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9183585000327775, + "recall": 0.9205277692543725, + "f1-score": 0.9166465058548297, + "support": 16295.0 + }, + "cross-entropy": 0.1804777281050399, + "roc-auc": 0.9616652728230258, + "score": 0.9616652728230258 + } + }, + "time": "0:03:06.389407", + "diversity": { + "train": { + "mean_centered_corr": 0.08821944147348404, + "mean_pairwise_disagreement": 0.03961999131425153, + "member_std": 0.03912428021430969 + }, + "val": { + "mean_centered_corr": 0.09983313083648682, + "mean_pairwise_disagreement": 0.03728577252442021, + "member_std": 0.04294782876968384 + }, + "test": { + "mean_centered_corr": 0.11663155257701874, + "mean_pairwise_disagreement": 0.04569306832691603, + "member_std": 0.04552939534187317 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1034e97f1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.9-greedy-heads-evaluation/0/report.json @@ -0,0 +1,307 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.946386196568126, + "recall": 0.9862395656450753, + "f1-score": 0.9659019657970156, + "support": 181971.0 + }, + "1": { + "precision": 0.9278094908608661, + "recall": 0.7599234928805875, + "f1-score": 0.8355163237489452, + "support": 42349.0 + }, + "accuracy": 0.9435137303851641, + "macro avg": { + "precision": 0.937097843714496, + "recall": 0.8730815292628313, + "f1-score": 0.9007091447729805, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9428791311704943, + "recall": 0.9435137303851641, + "f1-score": 0.9412866771241655, + "support": 224320.0 + }, + "cross-entropy": 0.13649569775840315, + "roc-auc": 0.9805235149654821, + "score": 0.9805235149654821 + }, + "val": { + "0": { + "precision": 0.9322954995754317, + "recall": 0.9837524640105131, + "f1-score": 0.9573330233098878, + "support": 16741.0 + }, + "1": { + "precision": 0.8900121310149616, + "recall": 0.6479246393876951, + "f1-score": 0.7499148211243611, + "support": 3397.0 + }, + "accuracy": 0.927102989373324, + "macro avg": { + "precision": 0.9111538152951966, + "recall": 0.815838551699104, + "f1-score": 0.8536239222171245, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9251628844696655, + "recall": 0.927102989373324, + "f1-score": 0.9223444627366316, + "support": 20138.0 + }, + "cross-entropy": 0.17444519969608893, + "roc-auc": 0.9587921238951639, + "score": 0.9587921238951639 + }, + "test": { + "0": { + "precision": 0.9292688218702538, + "recall": 0.9773332319160265, + "f1-score": 0.9526951879587752, + "support": 13147.0 + }, + "1": { + "precision": 0.8792544570502431, + "recall": 0.6893265565438373, + "f1-score": 0.7727920227920227, + "support": 3148.0 + }, + "accuracy": 0.921693771095428, + "macro avg": { + "precision": 0.9042616394602485, + "recall": 0.8333298942299319, + "f1-score": 0.862743605375399, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9196066420326721, + "recall": 0.921693771095428, + "f1-score": 0.9179400382843391, + "support": 16295.0 + }, + "cross-entropy": 0.1794710390286192, + "roc-auc": 0.9622146031450255, + "score": 0.9622146031450255 + } + }, + "time": "0:03:06.389407", + "diversity": { + "train": { + "mean_centered_corr": 0.08821944147348404, + "mean_pairwise_disagreement": 0.03961999131425153, + "member_std": 0.03912428021430969 + }, + "val": { + "mean_centered_corr": 0.09983313083648682, + "mean_pairwise_disagreement": 0.03728577252442021, + "member_std": 0.04294782876968384 + }, + "test": { + "mean_centered_corr": 0.11663155257701874, + "mean_pairwise_disagreement": 0.04569306832691603, + "member_std": 0.04552939534187317 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 7, + 23, + 22, + 15, + 5, + 2, + 30, + 7, + 28 + ], + "head_selection_time": "0:00:11.075623" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a5901fec9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-best-head-evaluation/0/report.json @@ -0,0 +1,298 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9506081938743591, + "min_features_per_member": 275.0, + "mean_features_per_member": 283.28125, + "max_features_per_member": 292.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.419462203979492, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9056525652610176, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9539299644574761, + "recall": 0.9690170411768908, + "f1-score": 0.9614143176489831, + "support": 181971.0 + }, + "1": { + "precision": 0.8571609536115122, + "recall": 0.7989090651491181, + "f1-score": 0.8270105108775361, + "support": 42349.0 + }, + "accuracy": 0.9369026390870185, + "macro avg": { + "precision": 0.9055454590344942, + "recall": 0.8839630531630045, + "f1-score": 0.8942124142632596, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9356611037214039, + "recall": 0.9369026390870185, + "f1-score": 0.9360404463358412, + "support": 224320.0 + }, + "cross-entropy": 0.14460702857706767, + "roc-auc": 0.9762667563935794, + "score": 0.9762667563935794 + }, + "val": { + "0": { + "precision": 0.9437383395522388, + "recall": 0.9669075921390597, + "f1-score": 0.9551824860590682, + "support": 16741.0 + }, + "1": { + "precision": 0.8144675150703282, + "recall": 0.7159258168972623, + "f1-score": 0.7620241265862447, + "support": 3397.0 + }, + "accuracy": 0.9245704637997815, + "macro avg": { + "precision": 0.8791029273112836, + "recall": 0.8414167045181611, + "f1-score": 0.8586033063226565, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9219321527032445, + "recall": 0.9245704637997815, + "f1-score": 0.9225993622568445, + "support": 20138.0 + }, + "cross-entropy": 0.1811185402192146, + "roc-auc": 0.9563975490624737, + "score": 0.9563975490624737 + }, + "test": { + "0": { + "precision": 0.9422366337380288, + "recall": 0.9504069369437894, + "f1-score": 0.9463041502574977, + "support": 13147.0 + }, + "1": { + "precision": 0.7851021753460777, + "recall": 0.7566709021601017, + "f1-score": 0.7706243934001941, + "support": 3148.0 + }, + "accuracy": 0.9129794415464867, + "macro avg": { + "precision": 0.8636694045420532, + "recall": 0.8535389195519456, + "f1-score": 0.8584642718288459, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.911880127139817, + "recall": 0.9129794415464867, + "f1-score": 0.9123649127866912, + "support": 16295.0 + }, + "cross-entropy": 0.19078130335337062, + "roc-auc": 0.9593891219693567, + "score": 0.9593891219693567 + } + }, + "time": "0:03:07.006918", + "diversity": { + "train": { + "mean_centered_corr": 0.08914346992969513, + "mean_pairwise_disagreement": 0.03838632562640928, + "member_std": 0.037065718322992325 + }, + "val": { + "mean_centered_corr": 0.09447728097438812, + "mean_pairwise_disagreement": 0.0372919796949436, + "member_std": 0.04158519208431244 + }, + "test": { + "mean_centered_corr": 0.11072365194559097, + "mean_pairwise_disagreement": 0.045909714042502645, + "member_std": 0.04358496516942978 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.952109" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0.toml new file mode 100644 index 000000000..49f80d6e4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0/report.json new file mode 100644 index 000000000..24e827ce9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0/report.json @@ -0,0 +1,294 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9506081938743591, + "min_features_per_member": 275.0, + "mean_features_per_member": 283.28125, + "max_features_per_member": 292.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.419462203979492, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9056525652610176, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9496583046818625, + "recall": 0.9835852965582428, + "f1-score": 0.9663241038432363, + "support": 181971.0 + }, + "1": { + "precision": 0.9166759651863423, + "recall": 0.7759569293253678, + "f1-score": 0.8404670255892169, + "support": 42349.0 + }, + "accuracy": 0.9443874821683309, + "macro avg": { + "precision": 0.9331671349341024, + "recall": 0.8797711129418053, + "f1-score": 0.9033955647162266, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9434316236222344, + "recall": 0.9443874821683309, + "f1-score": 0.9425637552029926, + "support": 224320.0 + }, + "cross-entropy": 0.1349038769118999, + "roc-auc": 0.9810848735528083, + "score": 0.9810848735528083 + }, + "val": { + "0": { + "precision": 0.9357175450604609, + "recall": 0.979929514365928, + "f1-score": 0.9573133370291483, + "support": 16741.0 + }, + "1": { + "precision": 0.8710667689946278, + "recall": 0.6682366794230203, + "f1-score": 0.756288522405464, + "support": 3397.0 + }, + "accuracy": 0.9273512761942596, + "macro avg": { + "precision": 0.9033921570275443, + "recall": 0.8240830968944741, + "f1-score": 0.8568009297173061, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.924811859972784, + "recall": 0.9273512761942596, + "f1-score": 0.9234032518530308, + "support": 20138.0 + }, + "cross-entropy": 0.17405502211594576, + "roc-auc": 0.9584246137411132, + "score": 0.9584246137411132 + }, + "test": { + "0": { + "precision": 0.9328956965718453, + "recall": 0.9728455160873203, + "f1-score": 0.9524518747440146, + "support": 13147.0 + }, + "1": { + "precision": 0.8618955512572534, + "recall": 0.7077509529860229, + "f1-score": 0.7772544915402059, + "support": 3148.0 + }, + "accuracy": 0.9216324025774778, + "macro avg": { + "precision": 0.8973956239145493, + "recall": 0.8402982345366716, + "f1-score": 0.8648531831421102, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9191793137887625, + "recall": 0.9216324025774778, + "f1-score": 0.9186058261201674, + "support": 16295.0 + }, + "cross-entropy": 0.1782989047843598, + "roc-auc": 0.962318960200698, + "score": 0.962318960200698 + } + }, + "time": "0:03:07.006918", + "diversity": { + "train": { + "mean_centered_corr": 0.08914346992969513, + "mean_pairwise_disagreement": 0.03838632562640928, + "member_std": 0.037065718322992325 + }, + "val": { + "mean_centered_corr": 0.09447728097438812, + "mean_pairwise_disagreement": 0.0372919796949436, + "member_std": 0.04158519208431244 + }, + "test": { + "mean_centered_corr": 0.11072365194559097, + "mean_pairwise_disagreement": 0.045909714042502645, + "member_std": 0.04358496516942978 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..633179f9a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-greedy-heads-evaluation/0/report.json @@ -0,0 +1,304 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 2874464, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9506081938743591, + "min_features_per_member": 275.0, + "mean_features_per_member": 283.28125, + "max_features_per_member": 292.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.419462203979492, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9056525652610176, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 2206 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9503279377423628, + "recall": 0.9817773161657627, + "f1-score": 0.9657966721086376, + "support": 181971.0 + }, + "1": { + "precision": 0.9087180334186693, + "recall": 0.779498925594465, + "f1-score": 0.8391631501347298, + "support": 42349.0 + }, + "accuracy": 0.943589514978602, + "macro avg": { + "precision": 0.929522985580516, + "recall": 0.8806381208801138, + "f1-score": 0.9024799111216837, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9424724730570735, + "recall": 0.943589514978602, + "f1-score": 0.941889739948897, + "support": 224320.0 + }, + "cross-entropy": 0.13479785159389124, + "roc-auc": 0.9805680231536485, + "score": 0.9805680231536485 + }, + "val": { + "0": { + "precision": 0.9373999542648068, + "recall": 0.9794516456603548, + "f1-score": 0.9579645371425233, + "support": 16741.0 + }, + "1": { + "precision": 0.8699924414210128, + "recall": 0.6776567559611422, + "f1-score": 0.7618732417673341, + "support": 3397.0 + }, + "accuracy": 0.9285430529347503, + "macro avg": { + "precision": 0.9036961978429099, + "recall": 0.8285542008107485, + "f1-score": 0.8599188894549287, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9260292460946623, + "recall": 0.9285430529347503, + "f1-score": 0.9248866679206782, + "support": 20138.0 + }, + "cross-entropy": 0.17265539939712965, + "roc-auc": 0.9592957095194116, + "score": 0.9592957095194116 + }, + "test": { + "0": { + "precision": 0.9348464815630968, + "recall": 0.9680535483380239, + "f1-score": 0.9511602705429543, + "support": 13147.0 + }, + "1": { + "precision": 0.8433420365535248, + "recall": 0.7182337992376112, + "f1-score": 0.7757762909589981, + "support": 3148.0 + }, + "accuracy": 0.919791347038969, + "macro avg": { + "precision": 0.8890942590583109, + "recall": 0.8431436737878175, + "f1-score": 0.8634682807509761, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9171689121927296, + "recall": 0.919791347038969, + "f1-score": 0.9172781737199843, + "support": 16295.0 + }, + "cross-entropy": 0.1790101347961027, + "roc-auc": 0.9620218603265258, + "score": 0.9620218603265258 + } + }, + "time": "0:03:07.006918", + "diversity": { + "train": { + "mean_centered_corr": 0.08914346992969513, + "mean_pairwise_disagreement": 0.03838632562640928, + "member_std": 0.037065718322992325 + }, + "val": { + "mean_centered_corr": 0.09447728097438812, + "mean_pairwise_disagreement": 0.0372919796949436, + "member_std": 0.04158519208431244 + }, + "test": { + "mean_centered_corr": 0.11072365194559097, + "mean_pairwise_disagreement": 0.045909714042502645, + "member_std": 0.04358496516942978 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 5, + 28, + 20, + 9, + 6, + 23 + ], + "head_selection_time": "0:00:08.820400" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..2b8177a57 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,634 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.935987075725548, + "recall": 0.9806232861280094, + "f1-score": 0.9577854114110891, + "support": 181971.0 + }, + "1": { + "precision": 0.8952808054408838, + "recall": 0.7118231835462466, + "f1-score": 0.7930807682188897, + "support": 42349.0 + }, + "accuracy": 0.9298769614835949, + "macro avg": { + "precision": 0.9156339405832159, + "recall": 0.846223234837128, + "f1-score": 0.8754330898149894, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9283022066087272, + "recall": 0.9298769614835949, + "f1-score": 0.9266910955473834, + "support": 224320.0 + }, + "cross-entropy": 0.16110536301253237, + "roc-auc": 0.9697341258683269, + "score": 0.9697341258683269 + }, + "val": { + "0": { + "precision": 0.9298739306618641, + "recall": 0.9869183441849352, + "f1-score": 0.9575473064997537, + "support": 16741.0 + }, + "1": { + "precision": 0.9075949367088607, + "recall": 0.6332057697968796, + "f1-score": 0.7459684411305705, + "support": 3397.0 + }, + "accuracy": 0.9272519614658854, + "macro avg": { + "precision": 0.9187344336853625, + "recall": 0.8100620569909074, + "f1-score": 0.8517578738151621, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9261157748142947, + "recall": 0.9272519614658854, + "f1-score": 0.9218569000214979, + "support": 20138.0 + }, + "cross-entropy": 0.17873116424917943, + "roc-auc": 0.9580215219221477, + "score": 0.9580215219221477 + }, + "test": { + "0": { + "precision": 0.9240352258895969, + "recall": 0.9816688217844375, + "f1-score": 0.9519805266651914, + "support": 13147.0 + }, + "1": { + "precision": 0.8964776632302406, + "recall": 0.6629606099110547, + "f1-score": 0.7622352081811541, + "support": 3148.0 + }, + "accuracy": 0.9200981896287205, + "macro avg": { + "precision": 0.9102564445599187, + "recall": 0.8223147158477461, + "f1-score": 0.8571078674231727, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9187114328701643, + "recall": 0.9200981896287205, + "f1-score": 0.9153239901455382, + "support": 16295.0 + }, + "cross-entropy": 0.18401642120758016, + "roc-auc": 0.9612981867919294, + "score": 0.9612981867919294 + } + }, + "time": "0:06:46.520241", + "diversity": { + "train": { + "mean_centered_corr": 0.018766077235341072, + "mean_pairwise_disagreement": 0.023167984748170816, + "member_std": 0.021088920533657074 + }, + "val": { + "mean_centered_corr": 0.017404615879058838, + "mean_pairwise_disagreement": 0.01965370395881322, + "member_std": 0.021817108616232872 + }, + "test": { + "mean_centered_corr": 0.028502995148301125, + "mean_pairwise_disagreement": 0.024374065862277167, + "member_std": 0.022561587393283844 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 5 + ], + "head_selection_time": "0:00:01.486676" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..8d11e7472 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "homesite-insurance" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..dafc0fcd1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-evaluation/0/report.json @@ -0,0 +1,630 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9356324122034465, + "recall": 0.9837171857054146, + "f1-score": 0.9590724737070513, + "support": 181971.0 + }, + "1": { + "precision": 0.9102039579355699, + "recall": 0.7092021063071147, + "f1-score": 0.7972287845406525, + "support": 42349.0 + }, + "accuracy": 0.9318919400855921, + "macro avg": { + "precision": 0.9229181850695082, + "recall": 0.8464596460062646, + "f1-score": 0.8781506291238519, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.930831816582056, + "recall": 0.9318919400855921, + "f1-score": 0.9285182725992239, + "support": 224320.0 + }, + "cross-entropy": 0.15573626126466394, + "roc-auc": 0.9717679542404869, + "score": 0.9717679542404869 + }, + "val": { + "0": { + "precision": 0.9283587121424959, + "recall": 0.9869183441849352, + "f1-score": 0.956743297237825, + "support": 16741.0 + }, + "1": { + "precision": 0.9064502349423323, + "recall": 0.6246688254342067, + "f1-score": 0.7396305332868596, + "support": 3397.0 + }, + "accuracy": 0.9258118979044593, + "macro avg": { + "precision": 0.9174044735424141, + "recall": 0.805793584809571, + "f1-score": 0.8481869152623422, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9246630573084033, + "recall": 0.9258118979044593, + "f1-score": 0.9201193991773706, + "support": 20138.0 + }, + "cross-entropy": 0.1749156826554181, + "roc-auc": 0.9584762673811861, + "score": 0.9584762673811861 + }, + "test": { + "0": { + "precision": 0.9240280160091481, + "recall": 0.9834182703278315, + "f1-score": 0.9527985555842146, + "support": 13147.0 + }, + "1": { + "precision": 0.9053408597481546, + "recall": 0.6623252858958069, + "f1-score": 0.7649972482113374, + "support": 3148.0 + }, + "accuracy": 0.9213869285056766, + "macro avg": { + "precision": 0.9146844378786514, + "recall": 0.8228717781118191, + "f1-score": 0.858897901897776, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9204178798993226, + "recall": 0.9213869285056766, + "f1-score": 0.9165175788668278, + "support": 16295.0 + }, + "cross-entropy": 0.17872381508706203, + "roc-auc": 0.9621948866927381, + "score": 0.9621948866927381 + } + }, + "time": "0:06:46.520241", + "diversity": { + "train": { + "mean_centered_corr": 0.018766077235341072, + "mean_pairwise_disagreement": 0.023167984748170816, + "member_std": 0.021088920533657074 + }, + "val": { + "mean_centered_corr": 0.017404615879058838, + "mean_pairwise_disagreement": 0.01965370395881322, + "member_std": 0.021817108616232872 + }, + "test": { + "mean_centered_corr": 0.028502995148301125, + "mean_pairwise_disagreement": 0.024374065862277167, + "member_std": 0.022561587393283844 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..cb053e547 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,637 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "homesite-insurance", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 7, + 8, + 4, + 8, + 7, + 2, + 21, + 16, + 12, + 8, + 3, + 3, + 10, + 4, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 298, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/homesite-insurance" + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9380125992360596, + "recall": 0.981095888905375, + "f1-score": 0.9590706419554123, + "support": 181971.0 + }, + "1": { + "precision": 0.8987967403136125, + "recall": 0.7214101867812699, + "f1-score": 0.800392978779146, + "support": 42349.0 + }, + "accuracy": 0.9320702567760343, + "macro avg": { + "precision": 0.918404669774836, + "recall": 0.8512530378433225, + "f1-score": 0.8797318103672791, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9306091024033799, + "recall": 0.9320702567760343, + "f1-score": 0.9291141496326069, + "support": 224320.0 + }, + "cross-entropy": 0.15562722826354536, + "roc-auc": 0.97165868278649, + "score": 0.97165868278649 + }, + "val": { + "0": { + "precision": 0.9312214128409283, + "recall": 0.9850666029508393, + "f1-score": 0.9573875181422351, + "support": 16741.0 + }, + "1": { + "precision": 0.8970769864141622, + "recall": 0.6414483367677363, + "f1-score": 0.7480260899416409, + "support": 3397.0 + }, + "accuracy": 0.927102989373324, + "macro avg": { + "precision": 0.9141491996275453, + "recall": 0.8132574698592878, + "f1-score": 0.852706804041938, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9254617238662671, + "recall": 0.927102989373324, + "f1-score": 0.9220711624168692, + "support": 20138.0 + }, + "cross-entropy": 0.1746060587875626, + "roc-auc": 0.9589731006657612, + "score": 0.9589731006657612 + }, + "test": { + "0": { + "precision": 0.9252939489532549, + "recall": 0.9816688217844375, + "f1-score": 0.9526480900535154, + "support": 13147.0 + }, + "1": { + "precision": 0.8973157221985514, + "recall": 0.6689961880559085, + "f1-score": 0.7665150136487716, + "support": 3148.0 + }, + "accuracy": 0.921264191469776, + "macro avg": { + "precision": 0.9113048355759031, + "recall": 0.825332504920173, + "f1-score": 0.8595815518511435, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9198888886388145, + "recall": 0.921264191469776, + "f1-score": 0.9166893956980607, + "support": 16295.0 + }, + "cross-entropy": 0.1790016934189047, + "roc-auc": 0.9623105275513741, + "score": 0.9623105275513741 + } + }, + "time": "0:06:46.520241", + "diversity": { + "train": { + "mean_centered_corr": 0.018766077235341072, + "mean_pairwise_disagreement": 0.023167984748170816, + "member_std": 0.021088920533657074 + }, + "val": { + "mean_centered_corr": 0.017404615879058838, + "mean_pairwise_disagreement": 0.01965370395881322, + "member_std": 0.021817108616232872 + }, + "test": { + "mean_centered_corr": 0.028502995148301125, + "mean_pairwise_disagreement": 0.024374065862277167, + "member_std": 0.022561587393283844 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 5, + 15, + 21, + 24 + ], + "head_selection_time": "0:00:08.828635" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ea0311c4d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-best-head-evaluation/0/report.json @@ -0,0 +1,285 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9441741289589772, + "recall": 0.9821235251770887, + "f1-score": 0.9627750123903721, + "support": 181971.0 + }, + "1": { + "precision": 0.9071499928642786, + "recall": 0.7504781694963282, + "f1-score": 0.8214101106171818, + "support": 42349.0 + }, + "accuracy": 0.9383915834522111, + "macro avg": { + "precision": 0.925662060911628, + "recall": 0.8663008473367084, + "f1-score": 0.8920925615037769, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9371844038364985, + "recall": 0.9383915834522111, + "f1-score": 0.9360869675205753, + "support": 224320.0 + }, + "cross-entropy": 0.14284985705983097, + "roc-auc": 0.9763333421966126, + "score": 0.9763333421966126 + }, + "val": { + "0": { + "precision": 0.9350575367437621, + "recall": 0.9804671166596978, + "f1-score": 0.9572240851436069, + "support": 16741.0 + }, + "1": { + "precision": 0.8734520123839009, + "recall": 0.6644097733294083, + "f1-score": 0.7547232904196622, + "support": 3397.0 + }, + "accuracy": 0.9271526467375112, + "macro avg": { + "precision": 0.9042547745638315, + "recall": 0.822438444994553, + "f1-score": 0.8559736877816346, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9246655431867828, + "recall": 0.9271526467375112, + "f1-score": 0.92306502269067, + "support": 20138.0 + }, + "cross-entropy": 0.18123454105665865, + "roc-auc": 0.9561241320583908, + "score": 0.9561241320583908 + }, + "test": { + "0": { + "precision": 0.9287627597191052, + "recall": 0.9758119723130753, + "f1-score": 0.9517062314540059, + "support": 13147.0 + }, + "1": { + "precision": 0.8718775181305399, + "recall": 0.6874205844980941, + "f1-score": 0.7687388987566608, + "support": 3148.0 + }, + "accuracy": 0.9200981896287205, + "macro avg": { + "precision": 0.9003201389248225, + "recall": 0.8316162784055847, + "f1-score": 0.8602225651053333, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9177732082910105, + "recall": 0.9200981896287205, + "f1-score": 0.9163591210930828, + "support": 16295.0 + }, + "cross-entropy": 0.1861704512569149, + "roc-auc": 0.960971584726283, + "score": 0.960971584726283 + } + }, + "time": "0:12:26.246333", + "diversity": { + "train": { + "mean_centered_corr": 0.055896881967782974, + "mean_pairwise_disagreement": 0.0307871046114997, + "member_std": 0.02823685109615326 + }, + "val": { + "mean_centered_corr": 0.06491804867982864, + "mean_pairwise_disagreement": 0.028161331970692545, + "member_std": 0.02979666367173195 + }, + "test": { + "mean_centered_corr": 0.08032963424921036, + "mean_pairwise_disagreement": 0.035900211820368406, + "member_std": 0.03238997608423233 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 12 + ], + "head_selection_time": "0:00:00.738599" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0.toml new file mode 100644 index 000000000..b0b8fc3b1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0/report.json new file mode 100644 index 000000000..4c2a205bb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/0/report.json @@ -0,0 +1,281 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9463842112495904, + "recall": 0.9839699732374939, + "f1-score": 0.9648111776576169, + "support": 181971.0 + }, + "1": { + "precision": 0.9169466431296623, + "recall": 0.7604665989751824, + "f1-score": 0.8314078816589433, + "support": 42349.0 + }, + "accuracy": 0.9417751426533524, + "macro avg": { + "precision": 0.9316654271896263, + "recall": 0.8722182861063381, + "f1-score": 0.8981095296582802, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9408267416868638, + "recall": 0.9417751426533524, + "f1-score": 0.9396261911105064, + "support": 224320.0 + }, + "cross-entropy": 0.1368376901407188, + "roc-auc": 0.9794268475505917, + "score": 0.9794268475505917 + }, + "val": { + "0": { + "precision": 0.9354489516864175, + "recall": 0.980765784600681, + "f1-score": 0.9575715160528387, + "support": 16741.0 + }, + "1": { + "precision": 0.8754833720030936, + "recall": 0.6664704150721225, + "f1-score": 0.7568109643991309, + "support": 3397.0 + }, + "accuracy": 0.9277485351077565, + "macro avg": { + "precision": 0.9054661618447555, + "recall": 0.8236180998364018, + "f1-score": 0.8571912402259848, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9253335939456164, + "recall": 0.9277485351077565, + "f1-score": 0.9237060083575539, + "support": 20138.0 + }, + "cross-entropy": 0.17396784432743392, + "roc-auc": 0.9581143402163179, + "score": 0.9581143402163179 + }, + "test": { + "0": { + "precision": 0.931127077436679, + "recall": 0.9758880352932228, + "f1-score": 0.9529822476416846, + "support": 13147.0 + }, + "1": { + "precision": 0.874006359300477, + "recall": 0.6985387547649301, + "f1-score": 0.7764830508474576, + "support": 3148.0 + }, + "accuracy": 0.9223074562749309, + "macro avg": { + "precision": 0.9025667183685779, + "recall": 0.8372133950290764, + "f1-score": 0.864732649244571, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9200920347430452, + "recall": 0.9223074562749309, + "f1-score": 0.9188847041309005, + "support": 16295.0 + }, + "cross-entropy": 0.17786783844384593, + "roc-auc": 0.962225790298713, + "score": 0.962225790298713 + } + }, + "time": "0:12:26.246333", + "diversity": { + "train": { + "mean_centered_corr": 0.055896881967782974, + "mean_pairwise_disagreement": 0.0307871046114997, + "member_std": 0.02823685109615326 + }, + "val": { + "mean_centered_corr": 0.06491804867982864, + "mean_pairwise_disagreement": 0.028161331970692545, + "member_std": 0.02979666367173195 + }, + "test": { + "mean_centered_corr": 0.08032963424921036, + "mean_pairwise_disagreement": 0.035900211820368406, + "member_std": 0.03238997608423233 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..fd4df6b14 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-greedy-heads-evaluation/0/report.json @@ -0,0 +1,295 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.943986898505545, + "recall": 0.9851294986563793, + "f1-score": 0.9641194708946333, + "support": 181971.0 + }, + "1": { + "precision": 0.9213783485385554, + "recall": 0.7488252379040827, + "f1-score": 0.8261883361340159, + "support": 42349.0 + }, + "accuracy": 0.9405180099857346, + "macro avg": { + "precision": 0.9326826235220502, + "recall": 0.8669773682802311, + "f1-score": 0.8951539035143246, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9397186679306875, + "recall": 0.9405180099857346, + "f1-score": 0.9380796811925229, + "support": 224320.0 + }, + "cross-entropy": 0.13886662867003355, + "roc-auc": 0.9784331962449399, + "score": 0.9784331962449399 + }, + "val": { + "0": { + "precision": 0.9338176864570326, + "recall": 0.9827369930111702, + "f1-score": 0.9576530166768532, + "support": 16741.0 + }, + "1": { + "precision": 0.8853174603174603, + "recall": 0.6567559611421843, + "f1-score": 0.7540983606557377, + "support": 3397.0 + }, + "accuracy": 0.9277485351077565, + "macro avg": { + "precision": 0.9095675733872464, + "recall": 0.8197464770766772, + "f1-score": 0.8558756886662955, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9256363741024727, + "recall": 0.9277485351077565, + "f1-score": 0.9233161825074357, + "support": 20138.0 + }, + "cross-entropy": 0.17316436914109928, + "roc-auc": 0.9589296236870106, + "score": 0.9589296236870106 + }, + "test": { + "0": { + "precision": 0.9291702804278693, + "recall": 0.9778656727770594, + "f1-score": 0.9528962680206056, + "support": 13147.0 + }, + "1": { + "precision": 0.8816592110614071, + "recall": 0.6886912325285895, + "f1-score": 0.773319065453897, + "support": 3148.0 + }, + "accuracy": 0.9220006136851795, + "macro avg": { + "precision": 0.9054147457446382, + "recall": 0.8332784526528245, + "f1-score": 0.8631076667372513, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.919991707468948, + "recall": 0.9220006136851795, + "f1-score": 0.9182040904397527, + "support": 16295.0 + }, + "cross-entropy": 0.17763494877994662, + "roc-auc": 0.9624745655349262, + "score": 0.9624745655349262 + } + }, + "time": "0:12:26.246333", + "diversity": { + "train": { + "mean_centered_corr": 0.055896881967782974, + "mean_pairwise_disagreement": 0.0307871046114997, + "member_std": 0.02823685109615326 + }, + "val": { + "mean_centered_corr": 0.06491804867982864, + "mean_pairwise_disagreement": 0.028161331970692545, + "member_std": 0.02979666367173195 + }, + "test": { + "mean_centered_corr": 0.08032963424921036, + "mean_pairwise_disagreement": 0.035900211820368406, + "member_std": 0.03238997608423233 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 12, + 31, + 24, + 17, + 15, + 29, + 21, + 13, + 24, + 17, + 5 + ], + "head_selection_time": "0:00:11.343819" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f4f8dc42f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,311 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9484513038850453, + "recall": 0.9793538530864808, + "f1-score": 0.9636548958961367, + "support": 181971.0 + }, + "1": { + "precision": 0.8968423942888523, + "recall": 0.7712814942501593, + "f1-score": 0.8293364140715256, + "support": 42349.0 + }, + "accuracy": 0.940072218259629, + "macro avg": { + "precision": 0.9226468490869488, + "recall": 0.8753176736683201, + "f1-score": 0.8964956549838312, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9387081436118231, + "recall": 0.940072218259629, + "f1-score": 0.9382971329379053, + "support": 224320.0 + }, + "cross-entropy": 0.1391744611707364, + "roc-auc": 0.9777392813930507, + "score": 0.9777392813930507 + }, + "val": { + "0": { + "precision": 0.9401502021952629, + "recall": 0.9721044143121678, + "f1-score": 0.9558603271563243, + "support": 16741.0 + }, + "1": { + "precision": 0.8348656294200849, + "recall": 0.6950250220783044, + "f1-score": 0.7585542168674699, + "support": 3397.0 + }, + "accuracy": 0.9253649816267753, + "macro avg": { + "precision": 0.887507915807674, + "recall": 0.8335647181952361, + "f1-score": 0.8572072720118971, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9223901617882077, + "recall": 0.9253649816267753, + "f1-score": 0.9225775355856004, + "support": 20138.0 + }, + "cross-entropy": 0.1813000791881102, + "roc-auc": 0.9554584990741118, + "score": 0.9554584990741118 + }, + "test": { + "0": { + "precision": 0.9373008226487809, + "recall": 0.961968509926219, + "f1-score": 0.9494744744744745, + "support": 13147.0 + }, + "1": { + "precision": 0.8215560314061384, + "recall": 0.7312579415501906, + "f1-score": 0.773781512605042, + "support": 3148.0 + }, + "accuracy": 0.9173979748389076, + "macro avg": { + "precision": 0.8794284270274597, + "recall": 0.8466132257382049, + "f1-score": 0.8616279935397583, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9149403069794444, + "recall": 0.9173979748389076, + "f1-score": 0.9155326859525368, + "support": 16295.0 + }, + "cross-entropy": 0.18597181508732155, + "roc-auc": 0.9598635611836792, + "score": 0.9598635611836792 + } + }, + "time": "0:07:36.712536", + "diversity": { + "train": { + "mean_centered_corr": 0.08504261821508408, + "mean_pairwise_disagreement": 0.037604994736781555, + "member_std": 0.03812075033783913 + }, + "val": { + "mean_centered_corr": 0.09174208343029022, + "mean_pairwise_disagreement": 0.03609920179791695, + "member_std": 0.04196608066558838 + }, + "test": { + "mean_centered_corr": 0.10859175771474838, + "mean_pairwise_disagreement": 0.04310061962406834, + "member_std": 0.044081997126340866 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 31 + ], + "head_selection_time": "0:00:01.202505" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0.toml new file mode 100644 index 000000000..9df7aee00 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0.toml @@ -0,0 +1,54 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0/report.json new file mode 100644 index 000000000..7f868dc23 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-evaluation/0/report.json @@ -0,0 +1,307 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9459871673503734, + "recall": 0.9884542042413351, + "f1-score": 0.9667545436711089, + "support": 181971.0 + }, + "1": { + "precision": 0.9385313048566413, + "recall": 0.7574913221091407, + "f1-score": 0.8383488612160096, + "support": 42349.0 + }, + "accuracy": 0.9448511055634807, + "macro avg": { + "precision": 0.9422592361035074, + "recall": 0.872972763175238, + "f1-score": 0.9025517024435592, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9445795874611657, + "recall": 0.9448511055634807, + "f1-score": 0.942513048279298, + "support": 224320.0 + }, + "cross-entropy": 0.13350397534467387, + "roc-auc": 0.9820799272842926, + "score": 0.9820799272842926 + }, + "val": { + "0": { + "precision": 0.931040325313453, + "recall": 0.9847082014216594, + "f1-score": 0.9571225360699045, + "support": 16741.0 + }, + "1": { + "precision": 0.8947368421052632, + "recall": 0.6405652045922873, + "f1-score": 0.7466117687424945, + "support": 3397.0 + }, + "accuracy": 0.9266560730956401, + "macro avg": { + "precision": 0.9128885837093581, + "recall": 0.8126367030069733, + "f1-score": 0.8518671524061995, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.924916433543753, + "recall": 0.9266560730956401, + "f1-score": 0.9216123028485712, + "support": 20138.0 + }, + "cross-entropy": 0.1756831254542049, + "roc-auc": 0.9581138918187615, + "score": 0.9581138918187615 + }, + "test": { + "0": { + "precision": 0.9275205538727823, + "recall": 0.9782459876777972, + "f1-score": 0.9522081960537518, + "support": 13147.0 + }, + "1": { + "precision": 0.8822560724578016, + "recall": 0.6807496823379924, + "f1-score": 0.768513537744307, + "support": 3148.0 + }, + "accuracy": 0.9207732433261737, + "macro avg": { + "precision": 0.904888313165292, + "recall": 0.8294978350078948, + "f1-score": 0.8603608668990295, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9187759949593512, + "recall": 0.9207732433261737, + "f1-score": 0.9167205750437406, + "support": 16295.0 + }, + "cross-entropy": 0.18070868511872268, + "roc-auc": 0.9616453389098677, + "score": 0.9616453389098677 + } + }, + "time": "0:07:36.712536", + "diversity": { + "train": { + "mean_centered_corr": 0.08504261821508408, + "mean_pairwise_disagreement": 0.037604994736781555, + "member_std": 0.03812075033783913 + }, + "val": { + "mean_centered_corr": 0.09174208343029022, + "mean_pairwise_disagreement": 0.03609920179791695, + "member_std": 0.04196608066558838 + }, + "test": { + "mean_centered_corr": 0.10859175771474838, + "mean_pairwise_disagreement": 0.04310061962406834, + "member_std": 0.044081997126340866 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..47304e5df --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,320 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.901321291923523, + "min_features_per_member": 256.0, + "mean_features_per_member": 268.59375, + "max_features_per_member": 279.0, + "min_members_per_feature": 24.0, + "mean_members_per_feature": 28.842281341552734, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8197222809700668, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9458629588665929, + "recall": 0.986920992905463, + "f1-score": 0.9659558789912893, + "support": 181971.0 + }, + "1": { + "precision": 0.9309143686502177, + "recall": 0.7572788023329948, + "f1-score": 0.8351671245719345, + "support": 42349.0 + }, + "accuracy": 0.9435672253922968, + "macro avg": { + "precision": 0.9383886637584054, + "recall": 0.8720998976192289, + "f1-score": 0.900561501781612, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9430408393628782, + "recall": 0.9435672253922968, + "f1-score": 0.9412644874038016, + "support": 224320.0 + }, + "cross-entropy": 0.13696330439392607, + "roc-auc": 0.9808254468336747, + "score": 0.9808254468336747 + }, + "val": { + "0": { + "precision": 0.9327445652173914, + "recall": 0.9841705991278896, + "f1-score": 0.957767766313036, + "support": 16741.0 + }, + "1": { + "precision": 0.8928860145513339, + "recall": 0.6502796585222255, + "f1-score": 0.7525123488332481, + "support": 3397.0 + }, + "accuracy": 0.9278478498361307, + "macro avg": { + "precision": 0.9128152898843627, + "recall": 0.8172251288250576, + "f1-score": 0.855140057573142, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9260209831033485, + "recall": 0.9278478498361307, + "f1-score": 0.9231440373837064, + "support": 20138.0 + }, + "cross-entropy": 0.1752850415945319, + "roc-auc": 0.9589065795694565, + "score": 0.9589065795694565 + }, + "test": { + "0": { + "precision": 0.9281277087546952, + "recall": 0.9773332319160265, + "f1-score": 0.9520951428253862, + "support": 13147.0 + }, + "1": { + "precision": 0.8784169726642187, + "recall": 0.6839263024142312, + "f1-score": 0.769065904625826, + "support": 3148.0 + }, + "accuracy": 0.9206505062902731, + "macro avg": { + "precision": 0.903272340709457, + "recall": 0.8306297671651288, + "f1-score": 0.8605805237256061, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.91852418637281, + "recall": 0.9206505062902731, + "f1-score": 0.9167360730584506, + "support": 16295.0 + }, + "cross-entropy": 0.18154574753822755, + "roc-auc": 0.961628401124263, + "score": 0.961628401124263 + } + }, + "time": "0:07:36.712536", + "diversity": { + "train": { + "mean_centered_corr": 0.08504261821508408, + "mean_pairwise_disagreement": 0.037604994736781555, + "member_std": 0.03812075033783913 + }, + "val": { + "mean_centered_corr": 0.09174208343029022, + "mean_pairwise_disagreement": 0.03609920179791695, + "member_std": 0.04196608066558838 + }, + "test": { + "mean_centered_corr": 0.10859175771474838, + "mean_pairwise_disagreement": 0.04310061962406834, + "member_std": 0.044081997126340866 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 31, + 4, + 28, + 22, + 29, + 2, + 30, + 5, + 26, + 6 + ], + "head_selection_time": "0:00:11.216798" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7fa0e6c12 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3057696, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9463394409095983, + "recall": 0.978353693720428, + "f1-score": 0.9620803138628147, + "support": 181971.0 + }, + "1": { + "precision": 0.8911668002099854, + "recall": 0.7616236510897542, + "f1-score": 0.8213185302131344, + "support": 42349.0 + }, + "accuracy": 0.9374375891583452, + "macro avg": { + "precision": 0.9187531205597919, + "recall": 0.8699886724050911, + "f1-score": 0.8916994220379746, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9359234897639674, + "recall": 0.9374375891583452, + "f1-score": 0.9355061306612262, + "support": 224320.0 + }, + "cross-entropy": 0.14418027001253625, + "roc-auc": 0.9761245839581788, + "score": 0.9761245839581788 + }, + "val": { + "0": { + "precision": 0.9370957586858222, + "recall": 0.9779583059554388, + "f1-score": 0.9570910791535134, + "support": 16741.0 + }, + "1": { + "precision": 0.8616422947131609, + "recall": 0.6764792463938769, + "f1-score": 0.7579155672823219, + "support": 3397.0 + }, + "accuracy": 0.927102989373324, + "macro avg": { + "precision": 0.8993690266994916, + "recall": 0.8272187761746579, + "f1-score": 0.8575033232179177, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9243678106713654, + "recall": 0.927102989373324, + "f1-score": 0.9234929455838224, + "support": 20138.0 + }, + "cross-entropy": 0.17790377870690235, + "roc-auc": 0.9559529233208351, + "score": 0.9559529233208351 + }, + "test": { + "0": { + "precision": 0.9330988495640068, + "recall": 0.9685859891990568, + "f1-score": 0.9505113085019035, + "support": 13147.0 + }, + "1": { + "precision": 0.8440332326283988, + "recall": 0.7099745870393901, + "f1-score": 0.7712215320910973, + "support": 3148.0 + }, + "accuracy": 0.9186253451979135, + "macro avg": { + "precision": 0.8885660410962029, + "recall": 0.8392802881192234, + "f1-score": 0.8608664202965004, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9158924327420801, + "recall": 0.9186253451979135, + "f1-score": 0.9158746582324209, + "support": 16295.0 + }, + "cross-entropy": 0.18384793846446465, + "roc-auc": 0.9601378276664159, + "score": 0.9601378276664159 + } + }, + "time": "0:04:18.975261", + "diversity": { + "train": { + "mean_centered_corr": 0.054093651473522186, + "mean_pairwise_disagreement": 0.03080082888500299, + "member_std": 0.028895577415823936 + }, + "val": { + "mean_centered_corr": 0.0648474395275116, + "mean_pairwise_disagreement": 0.027818936435370138, + "member_std": 0.030719202011823654 + }, + "test": { + "mean_centered_corr": 0.07701285928487778, + "mean_pairwise_disagreement": 0.03625110116897128, + "member_std": 0.03344917297363281 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.950216" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..e37dbdda4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..3adca43df --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-evaluation/0/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3057696, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9447221607027979, + "recall": 0.9845469882563705, + "f1-score": 0.9642235336223437, + "support": 181971.0 + }, + "1": { + "precision": 0.9189111252090663, + "recall": 0.7524616874070226, + "f1-score": 0.8273981850519948, + "support": 42349.0 + }, + "accuracy": 0.9407319900142653, + "macro avg": { + "precision": 0.931816642955932, + "recall": 0.8685043378316966, + "f1-score": 0.8958108593371692, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9398493382075944, + "recall": 0.9407319900142653, + "f1-score": 0.9383925034573752, + "support": 224320.0 + }, + "cross-entropy": 0.13832211550339593, + "roc-auc": 0.9790085111590701, + "score": 0.9790085111590701 + }, + "val": { + "0": { + "precision": 0.9330724292439453, + "recall": 0.9826772594229736, + "f1-score": 0.9572326312114512, + "support": 16741.0 + }, + "1": { + "precision": 0.8843238930993219, + "recall": 0.652634677656756, + "f1-score": 0.7510162601626016, + "support": 3397.0 + }, + "accuracy": 0.9270036746449498, + "macro avg": { + "precision": 0.9086981611716336, + "recall": 0.8176559685398648, + "f1-score": 0.8541244456870264, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9248492304514493, + "recall": 0.9270036746449498, + "f1-score": 0.92244680280481, + "support": 20138.0 + }, + "cross-entropy": 0.17414082376745435, + "roc-auc": 0.9582141658213201, + "score": 0.9582141658213201 + }, + "test": { + "0": { + "precision": 0.9287518948964123, + "recall": 0.978626302578535, + "f1-score": 0.953037037037037, + "support": 13147.0 + }, + "1": { + "precision": 0.8849303849303849, + "recall": 0.6864675984752223, + "f1-score": 0.7731663685152057, + "support": 3148.0 + }, + "accuracy": 0.9221847192390303, + "macro avg": { + "precision": 0.9068411399133987, + "recall": 0.8325469505268787, + "f1-score": 0.8631017027761214, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9202861008876332, + "recall": 0.9221847192390303, + "f1-score": 0.9182881653275111, + "support": 16295.0 + }, + "cross-entropy": 0.17803392077378757, + "roc-auc": 0.962170313614336, + "score": 0.962170313614336 + } + }, + "time": "0:04:18.975261", + "diversity": { + "train": { + "mean_centered_corr": 0.054093651473522186, + "mean_pairwise_disagreement": 0.03080082888500299, + "member_std": 0.028895577415823936 + }, + "val": { + "mean_centered_corr": 0.0648474395275116, + "mean_pairwise_disagreement": 0.027818936435370138, + "member_std": 0.030719202011823654 + }, + "test": { + "mean_centered_corr": 0.07701285928487778, + "mean_pairwise_disagreement": 0.03625110116897128, + "member_std": 0.03344917297363281 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ece4c009a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,286 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3057696, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.944993059477376, + "recall": 0.9839205148073045, + "f1-score": 0.9640639891448909, + "support": 181971.0 + }, + "1": { + "precision": 0.9160473990761197, + "recall": 0.7539020992231221, + "f1-score": 0.8271029247946944, + "support": 42349.0 + }, + "accuracy": 0.9404957203994294, + "macro avg": { + "precision": 0.9305202292767478, + "recall": 0.8689113070152132, + "f1-score": 0.8955834569697927, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9395284563553503, + "recall": 0.9404957203994294, + "f1-score": 0.9382073374233928, + "support": 224320.0 + }, + "cross-entropy": 0.1396240616739436, + "roc-auc": 0.9782459886492416, + "score": 0.9782459886492416 + }, + "val": { + "0": { + "precision": 0.9338289540888712, + "recall": 0.9829161937757601, + "f1-score": 0.9577440195564868, + "support": 16741.0 + }, + "1": { + "precision": 0.88637266587207, + "recall": 0.6567559611421843, + "f1-score": 0.7544808927967535, + "support": 3397.0 + }, + "accuracy": 0.9278975072003178, + "macro avg": { + "precision": 0.9101008099804706, + "recall": 0.8198360774589721, + "f1-score": 0.8561124561766201, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9258237395158017, + "recall": 0.9278975072003178, + "f1-score": 0.9234563623113375, + "support": 20138.0 + }, + "cross-entropy": 0.1730646434975347, + "roc-auc": 0.9589703663198782, + "score": 0.9589703663198782 + }, + "test": { + "0": { + "precision": 0.9299920445505171, + "recall": 0.9780938617175021, + "f1-score": 0.9534366426929636, + "support": 13147.0 + }, + "1": { + "precision": 0.8833063209076175, + "recall": 0.6925031766200762, + "f1-score": 0.7763532763532763, + "support": 3148.0 + }, + "accuracy": 0.9229211414544338, + "macro avg": { + "precision": 0.9066491827290672, + "recall": 0.8352985191687892, + "f1-score": 0.8648949595231199, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9209729185592407, + "recall": 0.9229211414544338, + "f1-score": 0.9192262445808228, + "support": 16295.0 + }, + "cross-entropy": 0.17706493879213278, + "roc-auc": 0.9625895781732687, + "score": 0.9625895781732687 + } + }, + "time": "0:04:18.975261", + "diversity": { + "train": { + "mean_centered_corr": 0.054093651473522186, + "mean_pairwise_disagreement": 0.03080082888500299, + "member_std": 0.028895577415823936 + }, + "val": { + "mean_centered_corr": 0.0648474395275116, + "mean_pairwise_disagreement": 0.027818936435370138, + "member_std": 0.030719202011823654 + }, + "test": { + "mean_centered_corr": 0.07701285928487778, + "mean_pairwise_disagreement": 0.03625110116897128, + "member_std": 0.03344917297363281 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 24, + 13, + 15, + 17, + 8, + 11, + 23, + 24, + 13, + 6, + 12 + ], + "head_selection_time": "0:00:12.138914" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a409eec65 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3057696, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9443811031934547, + "recall": 0.9831951244978595, + "f1-score": 0.9633973307917431, + "support": 181971.0 + }, + "1": { + "precision": 0.9123028391167193, + "recall": 0.7511865687501476, + "f1-score": 0.8239422939950012, + "support": 42349.0 + }, + "accuracy": 0.9393946148359487, + "macro avg": { + "precision": 0.928341971155087, + "recall": 0.8671908466240035, + "f1-score": 0.8936698123933722, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9383251010296455, + "recall": 0.9393946148359487, + "f1-score": 0.9370698461568233, + "support": 224320.0 + }, + "cross-entropy": 0.13960149767979568, + "roc-auc": 0.9778102042377117, + "score": 0.9778102042377117 + }, + "val": { + "0": { + "precision": 0.9351369902190699, + "recall": 0.9765844334269159, + "f1-score": 0.955411407199626, + "support": 16741.0 + }, + "1": { + "precision": 0.8523540489642184, + "recall": 0.6661760376803062, + "f1-score": 0.7478519497686715, + "support": 3397.0 + }, + "accuracy": 0.9242228622504718, + "macro avg": { + "precision": 0.8937455195916442, + "recall": 0.8213802355536111, + "f1-score": 0.8516316784841488, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9211726615149916, + "recall": 0.9242228622504718, + "f1-score": 0.9203990188346965, + "support": 20138.0 + }, + "cross-entropy": 0.17988717365824028, + "roc-auc": 0.9553321388843028, + "score": 0.9553321388843028 + }, + "test": { + "0": { + "precision": 0.9336227831334167, + "recall": 0.9650110291321214, + "f1-score": 0.9490574506283662, + "support": 13147.0 + }, + "1": { + "precision": 0.8300073909830007, + "recall": 0.7134688691232529, + "f1-score": 0.7673385719166382, + "support": 3148.0 + }, + "accuracy": 0.9164160785517029, + "macro avg": { + "precision": 0.8818150870582087, + "recall": 0.8392399491276872, + "f1-score": 0.8581980112725023, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9136055229622286, + "recall": 0.9164160785517029, + "f1-score": 0.9139515267140048, + "support": 16295.0 + }, + "cross-entropy": 0.1855192604147141, + "roc-auc": 0.9596838539362689, + "score": 0.9596838539362689 + } + }, + "time": "0:04:29.924516", + "diversity": { + "train": { + "mean_centered_corr": 0.05251581221818924, + "mean_pairwise_disagreement": 0.03326311814055957, + "member_std": 0.03071427531540394 + }, + "val": { + "mean_centered_corr": 0.06350768357515335, + "mean_pairwise_disagreement": 0.03187952722985593, + "member_std": 0.033710550516843796 + }, + "test": { + "mean_centered_corr": 0.07430540025234222, + "mean_pairwise_disagreement": 0.04065862277167942, + "member_std": 0.03548940643668175 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.832394" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..e20b29645 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..8511fd467 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-evaluation/0/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3057696, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9489507453837032, + "recall": 0.9850635540827934, + "f1-score": 0.966669992207449, + "support": 181971.0 + }, + "1": { + "precision": 0.9232723577235772, + "recall": 0.7722968665139672, + "f1-score": 0.8410630938757666, + "support": 42349.0 + }, + "accuracy": 0.9448956847360913, + "macro avg": { + "precision": 0.9361115515536402, + "recall": 0.8786802102983804, + "f1-score": 0.9038665430416077, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9441029652525571, + "recall": 0.9448956847360913, + "f1-score": 0.9429568746189663, + "support": 224320.0 + }, + "cross-entropy": 0.13130891322714677, + "roc-auc": 0.9818073079002588, + "score": 0.9818073079002588 + }, + "val": { + "0": { + "precision": 0.9361495923835585, + "recall": 0.9808852517770742, + "f1-score": 0.95799544950703, + "support": 16741.0 + }, + "1": { + "precision": 0.8767809010396611, + "recall": 0.6702973211657345, + "f1-score": 0.7597597597597597, + "support": 3397.0 + }, + "accuracy": 0.9284933955705631, + "macro avg": { + "precision": 0.9064652467116099, + "recall": 0.8255912864714043, + "f1-score": 0.8588776046333948, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9261349213886623, + "recall": 0.9284933955705631, + "f1-score": 0.9245558508342979, + "support": 20138.0 + }, + "cross-entropy": 0.17334674258328744, + "roc-auc": 0.958447068787368, + "score": 0.958447068787368 + }, + "test": { + "0": { + "precision": 0.9326930073491959, + "recall": 0.974975279531452, + "f1-score": 0.9533655634064708, + "support": 13147.0 + }, + "1": { + "precision": 0.8710815047021944, + "recall": 0.7061626429479034, + "f1-score": 0.78, + "support": 3148.0 + }, + "accuracy": 0.9230438784903344, + "macro avg": { + "precision": 0.9018872560256952, + "recall": 0.8405689612396777, + "f1-score": 0.8666827817032354, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9207903985530768, + "recall": 0.9230438784903344, + "f1-score": 0.9198734005587526, + "support": 16295.0 + }, + "cross-entropy": 0.17722974814070125, + "roc-auc": 0.9625908587761747, + "score": 0.9625908587761747 + } + }, + "time": "0:04:29.924516", + "diversity": { + "train": { + "mean_centered_corr": 0.05251581221818924, + "mean_pairwise_disagreement": 0.03326311814055957, + "member_std": 0.03071427531540394 + }, + "val": { + "mean_centered_corr": 0.06350768357515335, + "mean_pairwise_disagreement": 0.03187952722985593, + "member_std": 0.033710550516843796 + }, + "test": { + "mean_centered_corr": 0.07430540025234222, + "mean_pairwise_disagreement": 0.04065862277167942, + "member_std": 0.03548940643668175 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..afbe38f9e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,283 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3057696, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9464479664251896, + "recall": 0.986470371652626, + "f1-score": 0.9660448233087664, + "support": 181971.0 + }, + "1": { + "precision": 0.9289548104114965, + "recall": 0.7601596259651939, + "f1-score": 0.8361232679246263, + "support": 42349.0 + }, + "accuracy": 0.943745542082739, + "macro avg": { + "precision": 0.937701388418343, + "recall": 0.87331499880891, + "f1-score": 0.9010840456166964, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9431454625734426, + "recall": 0.943745542082739, + "f1-score": 0.9415171487859286, + "support": 224320.0 + }, + "cross-entropy": 0.1327637198235955, + "roc-auc": 0.9809282833493577, + "score": 0.9809282833493577 + }, + "val": { + "0": { + "precision": 0.9345985567361782, + "recall": 0.9824980586583836, + "f1-score": 0.9579499126383226, + "support": 16741.0 + }, + "1": { + "precision": 0.8846002363135093, + "recall": 0.6611716220194289, + "f1-score": 0.7567385444743935, + "support": 3397.0 + }, + "accuracy": 0.9282947661138147, + "macro avg": { + "precision": 0.9095993965248437, + "recall": 0.8218348403389062, + "f1-score": 0.8573442285563581, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9261645367502904, + "recall": 0.9282947661138147, + "f1-score": 0.924008358479376, + "support": 20138.0 + }, + "cross-entropy": 0.17278712862311624, + "roc-auc": 0.9591161746546817, + "score": 0.9591161746546817 + }, + "test": { + "0": { + "precision": 0.9311094996374184, + "recall": 0.9766486650946984, + "f1-score": 0.9533355607528678, + "support": 13147.0 + }, + "1": { + "precision": 0.8774451097804391, + "recall": 0.6982210927573063, + "f1-score": 0.7776401910490005, + "support": 3148.0 + }, + "accuracy": 0.9228597729364836, + "macro avg": { + "precision": 0.9042773047089288, + "recall": 0.8374348789260023, + "f1-score": 0.8654878759009341, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9207421784180401, + "recall": 0.9228597729364836, + "f1-score": 0.9193933070659839, + "support": 16295.0 + }, + "cross-entropy": 0.1777732713928136, + "roc-auc": 0.9623163265079293, + "score": 0.9623163265079293 + } + }, + "time": "0:04:29.924516", + "diversity": { + "train": { + "mean_centered_corr": 0.05251581221818924, + "mean_pairwise_disagreement": 0.03326311814055957, + "member_std": 0.03071427531540394 + }, + "val": { + "mean_centered_corr": 0.06350768357515335, + "mean_pairwise_disagreement": 0.03187952722985593, + "member_std": 0.033710550516843796 + }, + "test": { + "mean_centered_corr": 0.07430540025234222, + "mean_pairwise_disagreement": 0.04065862277167942, + "member_std": 0.03548940643668175 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 26, + 23, + 24, + 5, + 9, + 2, + 21, + 6 + ], + "head_selection_time": "0:00:10.170223" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..289c9b3b9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3057696, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9480838502035509, + "recall": 0.977765687939287, + "f1-score": 0.9626960358620167, + "support": 181971.0 + }, + "1": { + "precision": 0.8896103896103896, + "recall": 0.7699355356679024, + "f1-score": 0.8254579055961317, + "support": 42349.0 + }, + "accuracy": 0.9385297788873038, + "macro avg": { + "precision": 0.9188471199069703, + "recall": 0.8738506118035947, + "f1-score": 0.8940769707290742, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9370447427558878, + "recall": 0.9385297788873038, + "f1-score": 0.9367870773267547, + "support": 224320.0 + }, + "cross-entropy": 0.14147821261922355, + "roc-auc": 0.9765004386230669, + "score": 0.9765004386230669 + }, + "val": { + "0": { + "precision": 0.9363126573586633, + "recall": 0.977420703661669, + "f1-score": 0.9564251687757548, + "support": 16741.0 + }, + "1": { + "precision": 0.8580015026296018, + "recall": 0.6723579629084486, + "f1-score": 0.7539197887440172, + "support": 3397.0 + }, + "accuracy": 0.9259608699970205, + "macro avg": { + "precision": 0.8971570799941326, + "recall": 0.8248893332850589, + "f1-score": 0.855172478759886, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9231026567322546, + "recall": 0.9259608699970205, + "f1-score": 0.9222653328452844, + "support": 20138.0 + }, + "cross-entropy": 0.18154143168207998, + "roc-auc": 0.9558571420859494, + "score": 0.9558571420859494 + }, + "test": { + "0": { + "precision": 0.9311504166057594, + "recall": 0.9690423670799422, + "f1-score": 0.9497185880949719, + "support": 13147.0 + }, + "1": { + "precision": 0.8442403367776502, + "recall": 0.7007623888182973, + "f1-score": 0.7658392640166638, + "support": 3148.0 + }, + "accuracy": 0.9172138692850568, + "macro avg": { + "precision": 0.8876953766917048, + "recall": 0.8349023779491198, + "f1-score": 0.8577789260558178, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9143604238902707, + "recall": 0.9172138692850568, + "f1-score": 0.9141952918569531, + "support": 16295.0 + }, + "cross-entropy": 0.1895139723221945, + "roc-auc": 0.958774227194806, + "score": 0.958774227194806 + } + }, + "time": "0:04:19.140003", + "diversity": { + "train": { + "mean_centered_corr": 0.05099165812134743, + "mean_pairwise_disagreement": 0.03067784069992177, + "member_std": 0.028426973149180412 + }, + "val": { + "mean_centered_corr": 0.059033043682575226, + "mean_pairwise_disagreement": 0.02795969904433602, + "member_std": 0.03006986528635025 + }, + "test": { + "mean_centered_corr": 0.06953592598438263, + "mean_pairwise_disagreement": 0.03653245602747726, + "member_std": 0.032674212008714676 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 7 + ], + "head_selection_time": "0:00:00.881435" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..72e7526d1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..9cd91ee41 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-evaluation/0/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3057696, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9455209967307316, + "recall": 0.9838051118035291, + "f1-score": 0.9642832134873826, + "support": 181971.0 + }, + "1": { + "precision": 0.9157542666018695, + "recall": 0.7564287232284115, + "f1-score": 0.8285012285012285, + "support": 42349.0 + }, + "accuracy": 0.9408791012838802, + "macro avg": { + "precision": 0.9306376316663005, + "recall": 0.8701169175159703, + "f1-score": 0.8963922209943056, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9399013852193765, + "recall": 0.9408791012838802, + "f1-score": 0.9386491581994965, + "support": 224320.0 + }, + "cross-entropy": 0.13842434069038026, + "roc-auc": 0.9789222515567924, + "score": 0.9789222515567924 + }, + "val": { + "0": { + "precision": 0.9340303426331041, + "recall": 0.9819007227764172, + "f1-score": 0.957367501456028, + "support": 16741.0 + }, + "1": { + "precision": 0.8806616778259158, + "recall": 0.6582278481012658, + "f1-score": 0.7533692722371967, + "support": 3397.0 + }, + "accuracy": 0.9273016188300724, + "macro avg": { + "precision": 0.9073460102295099, + "recall": 0.8200642854388415, + "f1-score": 0.8553683868466124, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9250277925114427, + "recall": 0.9273016188300724, + "f1-score": 0.9229558426688411, + "support": 20138.0 + }, + "cross-entropy": 0.17369357261885118, + "roc-auc": 0.9583854185194205, + "score": 0.9583854185194205 + }, + "test": { + "0": { + "precision": 0.9296467863346844, + "recall": 0.9769529170152886, + "f1-score": 0.9527129770426139, + "support": 13147.0 + }, + "1": { + "precision": 0.8777732956837434, + "recall": 0.6912325285895807, + "f1-score": 0.7734138972809668, + "support": 3148.0 + }, + "accuracy": 0.9217551396133783, + "macro avg": { + "precision": 0.9037100410092139, + "recall": 0.8340927228024346, + "f1-score": 0.8630634371617903, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9196254455203756, + "recall": 0.9217551396133783, + "f1-score": 0.9180745294765099, + "support": 16295.0 + }, + "cross-entropy": 0.1775633543432209, + "roc-auc": 0.9623873274822505, + "score": 0.9623873274822505 + } + }, + "time": "0:04:19.140003", + "diversity": { + "train": { + "mean_centered_corr": 0.05099165812134743, + "mean_pairwise_disagreement": 0.03067784069992177, + "member_std": 0.028426973149180412 + }, + "val": { + "mean_centered_corr": 0.059033043682575226, + "mean_pairwise_disagreement": 0.02795969904433602, + "member_std": 0.03006986528635025 + }, + "test": { + "mean_centered_corr": 0.06953592598438263, + "mean_pairwise_disagreement": 0.03653245602747726, + "member_std": 0.032674212008714676 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9549e7e89 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,292 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3057696, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9446692233434791, + "recall": 0.98411285314693, + "f1-score": 0.9639877267588954, + "support": 181971.0 + }, + "1": { + "precision": 0.9168081494057725, + "recall": 0.7523200075562587, + "f1-score": 0.8264591439688715, + "support": 42349.0 + }, + "accuracy": 0.9403530670470756, + "macro avg": { + "precision": 0.9307386863746259, + "recall": 0.8682164303515943, + "f1-score": 0.8952234353638835, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9394093774974203, + "recall": 0.9403530670470756, + "f1-score": 0.9380239341743076, + "support": 224320.0 + }, + "cross-entropy": 0.13997136532599735, + "roc-auc": 0.978227078576783, + "score": 0.978227078576783 + }, + "val": { + "0": { + "precision": 0.9330876098667423, + "recall": 0.9829161937757601, + "f1-score": 0.9573539678845706, + "support": 16741.0 + }, + "1": { + "precision": 0.8857371154614463, + "recall": 0.652634677656756, + "f1-score": 0.7515254237288136, + "support": 3397.0 + }, + "accuracy": 0.9272023041016982, + "macro avg": { + "precision": 0.9094123626640943, + "recall": 0.8177754357162581, + "f1-score": 0.8544396958066921, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9251002412355581, + "recall": 0.9272023041016982, + "f1-score": 0.9226335604708698, + "support": 20138.0 + }, + "cross-entropy": 0.17266928890233252, + "roc-auc": 0.9591292133522524, + "score": 0.9591292133522524 + }, + "test": { + "0": { + "precision": 0.9302510672165545, + "recall": 0.977941735757207, + "f1-score": 0.9535004449718184, + "support": 13147.0 + }, + "1": { + "precision": 0.8827809215844786, + "recall": 0.6937738246505718, + "f1-score": 0.7769477054429029, + "support": 3148.0 + }, + "accuracy": 0.9230438784903344, + "macro avg": { + "precision": 0.9065159944005166, + "recall": 0.8358577802038893, + "f1-score": 0.8652240752073607, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9210804002358993, + "recall": 0.9230438784903344, + "f1-score": 0.9193925576421451, + "support": 16295.0 + }, + "cross-entropy": 0.17693632251110164, + "roc-auc": 0.9626703044809795, + "score": 0.9626703044809795 + } + }, + "time": "0:04:19.140003", + "diversity": { + "train": { + "mean_centered_corr": 0.05099165812134743, + "mean_pairwise_disagreement": 0.03067784069992177, + "member_std": 0.028426973149180412 + }, + "val": { + "mean_centered_corr": 0.059033043682575226, + "mean_pairwise_disagreement": 0.02795969904433602, + "member_std": 0.03006986528635025 + }, + "test": { + "mean_centered_corr": 0.06953592598438263, + "mean_pairwise_disagreement": 0.03653245602747726, + "member_std": 0.032674212008714676 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 7, + 31, + 15, + 17, + 24, + 0, + 6, + 19, + 21, + 29, + 5, + 17, + 12, + 24, + 31, + 13, + 15, + 0 + ], + "head_selection_time": "0:00:16.480622" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..2c5ded4f4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9470929244882358, + "recall": 0.9803869847393266, + "f1-score": 0.9634524044186542, + "support": 181971.0 + }, + "1": { + "precision": 0.9007287494437027, + "recall": 0.7646697678811778, + "f1-score": 0.8271414158184442, + "support": 42349.0 + }, + "accuracy": 0.9396620898716119, + "macro avg": { + "precision": 0.9239108369659692, + "recall": 0.8725283763102523, + "f1-score": 0.8952969101185493, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9383399089347366, + "recall": 0.9396620898716119, + "f1-score": 0.9377184794176276, + "support": 224320.0 + }, + "cross-entropy": 0.1374541129990154, + "roc-auc": 0.9783636099059336, + "score": 0.9783636099059336 + }, + "val": { + "0": { + "precision": 0.9391985260248733, + "recall": 0.9743742906636401, + "f1-score": 0.9564631035797003, + "support": 16741.0 + }, + "1": { + "precision": 0.8451263537906137, + "recall": 0.6891374742419782, + "f1-score": 0.7592022052862007, + "support": 3397.0 + }, + "accuracy": 0.9262588141821432, + "macro avg": { + "precision": 0.8921624399077435, + "recall": 0.8317558824528092, + "f1-score": 0.8578326544329505, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9233298613570919, + "recall": 0.9262588141821432, + "f1-score": 0.9231879386426154, + "support": 20138.0 + }, + "cross-entropy": 0.18034048370527975, + "roc-auc": 0.9557610707114682, + "score": 0.9557610707114682 + }, + "test": { + "0": { + "precision": 0.9347106830380119, + "recall": 0.9669886666159581, + "f1-score": 0.9505757439808584, + "support": 13147.0 + }, + "1": { + "precision": 0.8389012620638456, + "recall": 0.7179161372299873, + "f1-score": 0.7737076343717905, + "support": 3148.0 + }, + "accuracy": 0.9188708192697146, + "macro avg": { + "precision": 0.8868059725509287, + "recall": 0.8424524019229727, + "f1-score": 0.8621416891763245, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.916201443564144, + "recall": 0.9188708192697146, + "f1-score": 0.9164069309063357, + "support": 16295.0 + }, + "cross-entropy": 0.18574135097736744, + "roc-auc": 0.9604946447119459, + "score": 0.9604946447119459 + } + }, + "time": "0:07:11.833138", + "diversity": { + "train": { + "mean_centered_corr": 0.05171258747577667, + "mean_pairwise_disagreement": 0.03286417948437716, + "member_std": 0.03040449135005474 + }, + "val": { + "mean_centered_corr": 0.061806585639715195, + "mean_pairwise_disagreement": 0.03182976975001522, + "member_std": 0.0331539586186409 + }, + "test": { + "mean_centered_corr": 0.0732795000076294, + "mean_pairwise_disagreement": 0.03959877856852983, + "member_std": 0.03512473404407501 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.946516" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..d6037d45d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..e0392c3f8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-evaluation/0/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.9493414440624174, + "recall": 0.9862725379318683, + "f1-score": 0.9674546723770353, + "support": 181971.0 + }, + "1": { + "precision": 0.9291749362064077, + "recall": 0.7738553448723701, + "f1-score": 0.8444324198971901, + "support": 42349.0 + }, + "accuracy": 0.9461706490727532, + "macro avg": { + "precision": 0.9392581901344126, + "recall": 0.8800639414021192, + "f1-score": 0.9059435461371127, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9455342425592338, + "recall": 0.9461706490727532, + "f1-score": 0.9442295057834682, + "support": 224320.0 + }, + "cross-entropy": 0.1285332583359511, + "roc-auc": 0.9826472071386247, + "score": 0.9826472071386247 + }, + "val": { + "0": { + "precision": 0.9352083807788658, + "recall": 0.9811839197180575, + "f1-score": 0.9576446582130884, + "support": 16741.0 + }, + "1": { + "precision": 0.8776223776223776, + "recall": 0.6649985281130409, + "f1-score": 0.7566571763523698, + "support": 3397.0 + }, + "accuracy": 0.9278478498361307, + "macro avg": { + "precision": 0.9064153792006218, + "recall": 0.8230912239155492, + "f1-score": 0.8571509172827292, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9254944244414645, + "recall": 0.9278478498361307, + "f1-score": 0.9237408705538938, + "support": 20138.0 + }, + "cross-entropy": 0.173981665844509, + "roc-auc": 0.9583231528038466, + "score": 0.9583231528038466 + }, + "test": { + "0": { + "precision": 0.9317207815791385, + "recall": 0.9756598463527801, + "f1-score": 0.9531842163929554, + "support": 13147.0 + }, + "1": { + "precision": 0.8734177215189873, + "recall": 0.7013977128335451, + "f1-score": 0.7780126849894292, + "support": 3148.0 + }, + "accuracy": 0.9226756673826327, + "macro avg": { + "precision": 0.9025692515490629, + "recall": 0.8385287795931626, + "f1-score": 0.8655984506911922, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9204573245021606, + "recall": 0.9226756673826327, + "f1-score": 0.9193431620291443, + "support": 16295.0 + }, + "cross-entropy": 0.17786928687791626, + "roc-auc": 0.96242593886798, + "score": 0.96242593886798 + } + }, + "time": "0:07:11.833138", + "diversity": { + "train": { + "mean_centered_corr": 0.05171258747577667, + "mean_pairwise_disagreement": 0.03286417948437716, + "member_std": 0.03040449135005474 + }, + "val": { + "mean_centered_corr": 0.061806585639715195, + "mean_pairwise_disagreement": 0.03182976975001522, + "member_std": 0.0331539586186409 + }, + "test": { + "mean_centered_corr": 0.0732795000076294, + "mean_pairwise_disagreement": 0.03959877856852983, + "member_std": 0.03512473404407501 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..460c6ffd1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,285 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1980, + "metrics": { + "train": { + "0": { + "precision": 0.946255084192114, + "recall": 0.9869869374790489, + "f1-score": 0.9661919171541927, + "support": 181971.0 + }, + "1": { + "precision": 0.931394136052845, + "recall": 0.7591206403929255, + "f1-score": 0.836479542054251, + "support": 42349.0 + }, + "accuracy": 0.9439684379457918, + "macro avg": { + "precision": 0.9388246101224795, + "recall": 0.8730537889359872, + "f1-score": 0.9013357296042219, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9434495104904829, + "recall": 0.9439684379457918, + "f1-score": 0.9417037334295696, + "support": 224320.0 + }, + "cross-entropy": 0.1317045399668242, + "roc-auc": 0.9813354497094697, + "score": 0.9813354497094697 + }, + "val": { + "0": { + "precision": 0.9333144315282109, + "recall": 0.9831551281285467, + "f1-score": 0.9575866883872469, + "support": 16741.0 + }, + "1": { + "precision": 0.8873351977626848, + "recall": 0.6538121872240212, + "f1-score": 0.7528813559322034, + "support": 3397.0 + }, + "accuracy": 0.9275995630151952, + "macro avg": { + "precision": 0.9103248146454479, + "recall": 0.818483657676284, + "f1-score": 0.8552340221597252, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.925558375460007, + "recall": 0.9275995630151952, + "f1-score": 0.9230557511367858, + "support": 20138.0 + }, + "cross-entropy": 0.1731441036701711, + "roc-auc": 0.959029757015826, + "score": 0.959029757015826 + }, + "test": { + "0": { + "precision": 0.9292688218702538, + "recall": 0.9773332319160265, + "f1-score": 0.9526951879587752, + "support": 13147.0 + }, + "1": { + "precision": 0.8792544570502431, + "recall": 0.6893265565438373, + "f1-score": 0.7727920227920227, + "support": 3148.0 + }, + "accuracy": 0.921693771095428, + "macro avg": { + "precision": 0.9042616394602485, + "recall": 0.8333298942299319, + "f1-score": 0.862743605375399, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9196066420326721, + "recall": 0.921693771095428, + "f1-score": 0.9179400382843391, + "support": 16295.0 + }, + "cross-entropy": 0.17812450933232857, + "roc-auc": 0.9624579176971494, + "score": 0.9624579176971494 + } + }, + "time": "0:07:11.833138", + "diversity": { + "train": { + "mean_centered_corr": 0.05171258747577667, + "mean_pairwise_disagreement": 0.03286417948437716, + "member_std": 0.03040449135005474 + }, + "val": { + "mean_centered_corr": 0.061806585639715195, + "mean_pairwise_disagreement": 0.03182976975001522, + "member_std": 0.0331539586186409 + }, + "test": { + "mean_centered_corr": 0.0732795000076294, + "mean_pairwise_disagreement": 0.03959877856852983, + "member_std": 0.03512473404407501 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 22, + 6, + 20, + 26, + 21, + 24, + 7, + 22, + 13, + 5 + ], + "head_selection_time": "0:00:11.484544" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e1fb60dd6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9382483830460208, + "recall": 0.9813321902940578, + "f1-score": 0.9593067918710281, + "support": 181971.0 + }, + "1": { + "precision": 0.9000676609890271, + "recall": 0.7224727856619991, + "f1-score": 0.8015509156165675, + "support": 42349.0 + }, + "accuracy": 0.9324625534950072, + "macro avg": { + "precision": 0.919158022017524, + "recall": 0.8519024879780285, + "f1-score": 0.8804288537437979, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9310403079818641, + "recall": 0.9324625534950072, + "f1-score": 0.9295243221692621, + "support": 224320.0 + }, + "cross-entropy": 0.15611302140645272, + "roc-auc": 0.9712153076924243, + "score": 0.9712153076924243 + }, + "val": { + "0": { + "precision": 0.9314890246662141, + "recall": 0.9835135296577265, + "f1-score": 0.956794607316152, + "support": 16741.0 + }, + "1": { + "precision": 0.8878960194963444, + "recall": 0.6435089785104504, + "f1-score": 0.7462024236217785, + "support": 3397.0 + }, + "accuracy": 0.926159499453769, + "macro avg": { + "precision": 0.9096925220812793, + "recall": 0.8135112540840885, + "f1-score": 0.8514985154689652, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9241354921127308, + "recall": 0.926159499453769, + "f1-score": 0.9212706402881558, + "support": 20138.0 + }, + "cross-entropy": 0.17654285581424872, + "roc-auc": 0.9576809947504603, + "score": 0.9576809947504603 + }, + "test": { + "0": { + "precision": 0.9285455991899906, + "recall": 0.9765726021145509, + "f1-score": 0.9519537332245867, + "support": 13147.0 + }, + "1": { + "precision": 0.8752025931928687, + "recall": 0.6861499364675985, + "f1-score": 0.7692307692307693, + "support": 3148.0 + }, + "accuracy": 0.9204664007364222, + "macro avg": { + "precision": 0.9018740961914297, + "recall": 0.8313612692910747, + "f1-score": 0.860592251227678, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9182403655061034, + "recall": 0.9204664007364222, + "f1-score": 0.9166538319878553, + "support": 16295.0 + }, + "cross-entropy": 0.18087018625189427, + "roc-auc": 0.9611084763444616, + "score": 0.9611084763444616 + } + }, + "time": "0:06:04.606143", + "diversity": { + "train": { + "mean_centered_corr": 0.02503632754087448, + "mean_pairwise_disagreement": 0.025175350737425797, + "member_std": 0.02292787656188011 + }, + "val": { + "mean_centered_corr": 0.025311479344964027, + "mean_pairwise_disagreement": 0.021320028897382257, + "member_std": 0.0236747395247221 + }, + "test": { + "mean_centered_corr": 0.034202832728624344, + "mean_pairwise_disagreement": 0.02758341664274614, + "member_std": 0.0252092145383358 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 5 + ], + "head_selection_time": "0:00:01.295715" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..37f311a6f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..024a124ef --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-evaluation/0/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9370407971029661, + "recall": 0.984008440905419, + "f1-score": 0.9599504641307233, + "support": 181971.0 + }, + "1": { + "precision": 0.91242325749368, + "recall": 0.7159082859099388, + "f1-score": 0.8023075803485187, + "support": 42349.0 + }, + "accuracy": 0.9333942582025677, + "macro avg": { + "precision": 0.924732027298323, + "recall": 0.8499583634076788, + "f1-score": 0.881129022239621, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9323932927123024, + "recall": 0.9333942582025677, + "f1-score": 0.930189326090011, + "support": 224320.0 + }, + "cross-entropy": 0.15323431377042523, + "roc-auc": 0.9727788043541361, + "score": 0.9727788043541361 + }, + "val": { + "0": { + "precision": 0.9285593744726331, + "recall": 0.9860223403619855, + "f1-score": 0.9564285300422968, + "support": 16741.0 + }, + "1": { + "precision": 0.9008894536213469, + "recall": 0.6261407123932882, + "f1-score": 0.7387981938172977, + "support": 3397.0 + }, + "accuracy": 0.9253153242625881, + "macro avg": { + "precision": 0.91472441404699, + "recall": 0.8060815263776369, + "f1-score": 0.8476133619297972, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9238918443737247, + "recall": 0.9253153242625881, + "f1-score": 0.9197173247509908, + "support": 20138.0 + }, + "cross-entropy": 0.1750501462455179, + "roc-auc": 0.9581143050478821, + "score": 0.9581143050478821 + }, + "test": { + "0": { + "precision": 0.9257745266781411, + "recall": 0.9818970107248802, + "f1-score": 0.9530102247979033, + "support": 13147.0 + }, + "1": { + "precision": 0.8987664823479371, + "recall": 0.6712198221092758, + "f1-score": 0.7685033642480451, + "support": 3148.0 + }, + "accuracy": 0.9218778766492789, + "macro avg": { + "precision": 0.912270504513039, + "recall": 0.8265584164170781, + "f1-score": 0.8607567945229742, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9205568940576144, + "recall": 0.9218778766492789, + "f1-score": 0.917365695984712, + "support": 16295.0 + }, + "cross-entropy": 0.1786217717894183, + "roc-auc": 0.9619551240015043, + "score": 0.9619551240015043 + } + }, + "time": "0:06:04.606143", + "diversity": { + "train": { + "mean_centered_corr": 0.02503632754087448, + "mean_pairwise_disagreement": 0.025175350737425797, + "member_std": 0.02292787656188011 + }, + "val": { + "mean_centered_corr": 0.025311479344964027, + "mean_pairwise_disagreement": 0.021320028897382257, + "member_std": 0.0236747395247221 + }, + "test": { + "mean_centered_corr": 0.034202832728624344, + "mean_pairwise_disagreement": 0.02758341664274614, + "member_std": 0.0252092145383358 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..df299eeba --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,278 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9377624907587523, + "recall": 0.9828544108676657, + "f1-score": 0.9597791205606775, + "support": 181971.0 + }, + "1": { + "precision": 0.9071400934551623, + "recall": 0.7197100285721032, + "f1-score": 0.8026281139727182, + "support": 42349.0 + }, + "accuracy": 0.933175820256776, + "macro avg": { + "precision": 0.9224512921069573, + "recall": 0.8512822197198844, + "f1-score": 0.8812036172666978, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9319813392635236, + "recall": 0.933175820256776, + "f1-score": 0.9301108431980104, + "support": 224320.0 + }, + "cross-entropy": 0.154500471418488, + "roc-auc": 0.9722444215234017, + "score": 0.9722444215234017 + }, + "val": { + "0": { + "precision": 0.9297020221934321, + "recall": 0.9859028731855922, + "f1-score": 0.9569780251637966, + "support": 16741.0 + }, + "1": { + "precision": 0.9010482180293501, + "recall": 0.632617015013247, + "f1-score": 0.7433414043583535, + "support": 3397.0 + }, + "accuracy": 0.9263084715463303, + "macro avg": { + "precision": 0.9153751201113911, + "recall": 0.8092599440994196, + "f1-score": 0.8501597147610751, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9248685246889436, + "recall": 0.9263084715463303, + "f1-score": 0.9209405040159125, + "support": 20138.0 + }, + "cross-entropy": 0.17378638383137032, + "roc-auc": 0.9589014449778305, + "score": 0.9589014449778305 + }, + "test": { + "0": { + "precision": 0.9251968503937008, + "recall": 0.9831140184072412, + "f1-score": 0.9532765423903824, + "support": 13147.0 + }, + "1": { + "precision": 0.9045161290322581, + "recall": 0.6680432020330368, + "f1-score": 0.7684999086424265, + "support": 3148.0 + }, + "accuracy": 0.9222460877569807, + "macro avg": { + "precision": 0.9148564897129794, + "recall": 0.825578610220139, + "f1-score": 0.8608882255164044, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9212015812408428, + "recall": 0.9222460877569807, + "f1-score": 0.9175798966071014, + "support": 16295.0 + }, + "cross-entropy": 0.17739934033244287, + "roc-auc": 0.9626523760402965, + "score": 0.9626523760402965 + } + }, + "time": "0:06:04.606143", + "diversity": { + "train": { + "mean_centered_corr": 0.02503632754087448, + "mean_pairwise_disagreement": 0.025175350737425797, + "member_std": 0.02292787656188011 + }, + "val": { + "mean_centered_corr": 0.025311479344964027, + "mean_pairwise_disagreement": 0.021320028897382257, + "member_std": 0.0236747395247221 + }, + "test": { + "mean_centered_corr": 0.034202832728624344, + "mean_pairwise_disagreement": 0.02758341664274614, + "member_std": 0.0252092145383358 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 5, + 31, + 3, + 21 + ], + "head_selection_time": "0:00:07.034698" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7721a02fc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9413658167129513, + "recall": 0.9835633150337142, + "f1-score": 0.9620020478421065, + "support": 181971.0 + }, + "1": { + "precision": 0.9125233972859148, + "recall": 0.7367588372806914, + "f1-score": 0.8152754732757607, + "support": 42349.0 + }, + "accuracy": 0.9369695078459344, + "macro avg": { + "precision": 0.926944606999433, + "recall": 0.8601610761572027, + "f1-score": 0.8886387605589336, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9359207042828712, + "recall": 0.9369695078459344, + "f1-score": 0.9343017816763158, + "support": 224320.0 + }, + "cross-entropy": 0.14393086493300278, + "roc-auc": 0.9759690841627111, + "score": 0.9759690841627111 + }, + "val": { + "0": { + "precision": 0.9325333937061354, + "recall": 0.9841705991278896, + "f1-score": 0.9576564270975617, + "support": 16741.0 + }, + "1": { + "precision": 0.8927125506072875, + "recall": 0.6491021489549602, + "f1-score": 0.7516618373956026, + "support": 3397.0 + }, + "accuracy": 0.9276492203793822, + "macro avg": { + "precision": 0.9126229721567114, + "recall": 0.8166363740414249, + "f1-score": 0.8546591322465822, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.925816172333269, + "recall": 0.9276492203793822, + "f1-score": 0.9229080101138715, + "support": 20138.0 + }, + "cross-entropy": 0.17925081582806562, + "roc-auc": 0.9560453459701026, + "score": 0.9560453459701026 + }, + "test": { + "0": { + "precision": 0.9250304899921085, + "recall": 0.9807560660226667, + "f1-score": 0.9520785645721037, + "support": 13147.0 + }, + "1": { + "precision": 0.8926146010186757, + "recall": 0.6680432020330368, + "f1-score": 0.764171511627907, + "support": 3148.0 + }, + "accuracy": 0.9203436637005217, + "macro avg": { + "precision": 0.9088225455053921, + "recall": 0.8243996340278518, + "f1-score": 0.8581250381000054, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9187681261695637, + "recall": 0.9203436637005217, + "f1-score": 0.9157771590692911, + "support": 16295.0 + }, + "cross-entropy": 0.18465226452577868, + "roc-auc": 0.9600924508313724, + "score": 0.9600924508313724 + } + }, + "time": "0:06:58.103811", + "diversity": { + "train": { + "mean_centered_corr": 0.05266324058175087, + "mean_pairwise_disagreement": 0.03131294111810317, + "member_std": 0.028657343238592148 + }, + "val": { + "mean_centered_corr": 0.06352649629116058, + "mean_pairwise_disagreement": 0.02856399712948398, + "member_std": 0.030144035816192627 + }, + "test": { + "mean_centered_corr": 0.0760873481631279, + "mean_pairwise_disagreement": 0.03589971691296558, + "member_std": 0.032248713076114655 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.778477" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..601f75de8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..4bb9206ba --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9463105683240804, + "recall": 0.9844810436827847, + "f1-score": 0.9650185034555944, + "support": 181971.0 + }, + "1": { + "precision": 0.9193350281356223, + "recall": 0.7599943328059694, + "f1-score": 0.8321052767651698, + "support": 42349.0 + }, + "accuracy": 0.9421005706134095, + "macro avg": { + "precision": 0.9328227982298514, + "recall": 0.872237688244377, + "f1-score": 0.898561890110382, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9412179009228634, + "recall": 0.9421005706134095, + "f1-score": 0.9399260362787364, + "support": 224320.0 + }, + "cross-entropy": 0.1363573248497679, + "roc-auc": 0.9795959593437451, + "score": 0.9795959593437451 + }, + "val": { + "0": { + "precision": 0.9344513928368391, + "recall": 0.9818409891882205, + "f1-score": 0.9575602225393959, + "support": 16741.0 + }, + "1": { + "precision": 0.880690737833595, + "recall": 0.6605828672357963, + "f1-score": 0.7549201009251472, + "support": 3397.0 + }, + "accuracy": 0.9276492203793822, + "macro avg": { + "precision": 0.907571065335217, + "recall": 0.8212119282120084, + "f1-score": 0.8562401617322715, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9253827194310382, + "recall": 0.9276492203793822, + "f1-score": 0.9233776575814258, + "support": 20138.0 + }, + "cross-entropy": 0.17415890290359384, + "roc-auc": 0.9581626616470993, + "score": 0.9581626616470993 + }, + "test": { + "0": { + "precision": 0.9295326291419477, + "recall": 0.9772571689358789, + "f1-score": 0.9527976565686529, + "support": 13147.0 + }, + "1": { + "precision": 0.879094217549535, + "recall": 0.6905972045743329, + "f1-score": 0.7735278420209927, + "support": 3148.0 + }, + "accuracy": 0.9218778766492789, + "macro avg": { + "precision": 0.9043134233457413, + "recall": 0.8339271867551059, + "f1-score": 0.8631627492948228, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9197885285164236, + "recall": 0.9218778766492789, + "f1-score": 0.9181648626934744, + "support": 16295.0 + }, + "cross-entropy": 0.17842812985458048, + "roc-auc": 0.9620964494052155, + "score": 0.9620964494052155 + } + }, + "time": "0:06:58.103811", + "diversity": { + "train": { + "mean_centered_corr": 0.05266324058175087, + "mean_pairwise_disagreement": 0.03131294111810317, + "member_std": 0.028657343238592148 + }, + "val": { + "mean_centered_corr": 0.06352649629116058, + "mean_pairwise_disagreement": 0.02856399712948398, + "member_std": 0.030144035816192627 + }, + "test": { + "mean_centered_corr": 0.0760873481631279, + "mean_pairwise_disagreement": 0.03589971691296558, + "member_std": 0.032248713076114655 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b14bbaa40 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,284 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 3424160, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9441162844567987, + "recall": 0.985503184573366, + "f1-score": 0.9643658968753949, + "support": 181971.0 + }, + "1": { + "precision": 0.923251483765856, + "recall": 0.749344730690217, + "f1-score": 0.8272572046766856, + "support": 42349.0 + }, + "accuracy": 0.9409192225392297, + "macro avg": { + "precision": 0.9336838841113273, + "recall": 0.8674239576317915, + "f1-score": 0.8958115507760402, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9401772534098091, + "recall": 0.9409192225392297, + "f1-score": 0.9384813747377204, + "support": 224320.0 + }, + "cross-entropy": 0.13816220002148916, + "roc-auc": 0.9787552331834622, + "score": 0.9787552331834622 + }, + "val": { + "0": { + "precision": 0.931622965641953, + "recall": 0.984767935009856, + "f1-score": 0.9574585474925226, + "support": 16741.0 + }, + "1": { + "precision": 0.8955773955773956, + "recall": 0.6438033559022667, + "f1-score": 0.7491008734372324, + "support": 3397.0 + }, + "accuracy": 0.9272519614658854, + "macro avg": { + "precision": 0.9136001806096743, + "recall": 0.8142856454560614, + "f1-score": 0.8532797104648775, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.925542580225859, + "recall": 0.9272519614658854, + "f1-score": 0.9223115111053034, + "support": 20138.0 + }, + "cross-entropy": 0.17350924991795408, + "roc-auc": 0.9591329763748824, + "score": 0.9591329763748824 + }, + "test": { + "0": { + "precision": 0.9269169903610991, + "recall": 0.9801475621814862, + "f1-score": 0.9527893822322452, + "support": 13147.0 + }, + "1": { + "precision": 0.8909318846636022, + "recall": 0.6772554002541296, + "f1-score": 0.769536184804187, + "support": 3148.0 + }, + "accuracy": 0.9216324025774778, + "macro avg": { + "precision": 0.9089244375123506, + "recall": 0.828701481217808, + "f1-score": 0.8611627835182161, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9199650963607481, + "recall": 0.9216324025774778, + "f1-score": 0.9173870462087087, + "support": 16295.0 + }, + "cross-entropy": 0.17845179542990083, + "roc-auc": 0.962370812537228, + "score": 0.962370812537228 + } + }, + "time": "0:06:58.103811", + "diversity": { + "train": { + "mean_centered_corr": 0.05266324058175087, + "mean_pairwise_disagreement": 0.03131294111810317, + "member_std": 0.028657343238592148 + }, + "val": { + "mean_centered_corr": 0.06352649629116058, + "mean_pairwise_disagreement": 0.02856399712948398, + "member_std": 0.030144035816192627 + }, + "test": { + "mean_centered_corr": 0.0760873481631279, + "mean_pairwise_disagreement": 0.03589971691296558, + "member_std": 0.032248713076114655 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 24, + 30, + 13, + 17, + 21, + 28, + 24, + 6, + 8 + ], + "head_selection_time": "0:00:10.755431" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..eeca7f754 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 4157088, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9368638960222211, + "recall": 0.9823708173280358, + "f1-score": 0.9590778499977198, + "support": 181971.0 + }, + "1": { + "precision": 0.9042673828707848, + "recall": 0.7155304729745685, + "f1-score": 0.7989032283578745, + "support": 42349.0 + }, + "accuracy": 0.9319944721825963, + "macro avg": { + "precision": 0.9205656394465029, + "recall": 0.8489506451513021, + "f1-score": 0.8789905391777972, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9307100544768834, + "recall": 0.9319944721825963, + "f1-score": 0.9288387493743879, + "support": 224320.0 + }, + "cross-entropy": 0.15546864018533366, + "roc-auc": 0.9714250727707411, + "score": 0.9714250727707411 + }, + "val": { + "0": { + "precision": 0.9298532731376975, + "recall": 0.9842303327160863, + "f1-score": 0.956269405995183, + "support": 16741.0 + }, + "1": { + "precision": 0.890818858560794, + "recall": 0.6340889019723285, + "f1-score": 0.7408426483233018, + "support": 3397.0 + }, + "accuracy": 0.9251663521700269, + "macro avg": { + "precision": 0.9103360658492458, + "recall": 0.8091596173442074, + "f1-score": 0.8485560271592424, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.923268711298501, + "recall": 0.9251663521700269, + "f1-score": 0.9199299137014406, + "support": 20138.0 + }, + "cross-entropy": 0.17668222837030997, + "roc-auc": 0.9580729733437149, + "score": 0.9580729733437149 + }, + "test": { + "0": { + "precision": 0.9272727272727272, + "recall": 0.9775614208564691, + "f1-score": 0.9517532491576258, + "support": 13147.0 + }, + "1": { + "precision": 0.8788501026694046, + "recall": 0.6797966963151207, + "f1-score": 0.7666129321153502, + "support": 3148.0 + }, + "accuracy": 0.9200368211107701, + "macro avg": { + "precision": 0.9030614149710658, + "recall": 0.8286790585857949, + "f1-score": 0.859183090636488, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9179180526945585, + "recall": 0.9200368211107701, + "f1-score": 0.9159863440917109, + "support": 16295.0 + }, + "cross-entropy": 0.18111153198277266, + "roc-auc": 0.9613215034297445, + "score": 0.9613215034297445 + } + }, + "time": "0:10:44.579894", + "diversity": { + "train": { + "mean_centered_corr": 0.02127973735332489, + "mean_pairwise_disagreement": 0.024082001590469835, + "member_std": 0.02185266837477684 + }, + "val": { + "mean_centered_corr": 0.021363921463489532, + "mean_pairwise_disagreement": 0.020554144147318983, + "member_std": 0.022689195349812508 + }, + "test": { + "mean_centered_corr": 0.031094567850232124, + "mean_pairwise_disagreement": 0.026469256352136514, + "member_std": 0.02432551421225071 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 5 + ], + "head_selection_time": "0:00:01.310251" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..63cde1ed1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..1bcfacc8c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-evaluation/0/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 4157088, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9372981517637021, + "recall": 0.9840469085733441, + "f1-score": 0.9601038019612994, + "support": 181971.0 + }, + "1": { + "precision": 0.9127520812670934, + "recall": 0.7171361779498926, + "f1-score": 0.8032054164132131, + "support": 42349.0 + }, + "accuracy": 0.9336572753209701, + "macro avg": { + "precision": 0.9250251165153978, + "recall": 0.8505915432616183, + "f1-score": 0.8816546091872562, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9326641399080455, + "recall": 0.9336572753209701, + "f1-score": 0.9304832165049162, + "support": 224320.0 + }, + "cross-entropy": 0.15236869424446178, + "roc-auc": 0.9730574685406277, + "score": 0.9730574685406277 + }, + "val": { + "0": { + "precision": 0.9290057425965544, + "recall": 0.9856639388328057, + "f1-score": 0.9564965365330551, + "support": 16741.0 + }, + "1": { + "precision": 0.898989898989899, + "recall": 0.6287901089196349, + "f1-score": 0.7399965355967435, + "support": 3397.0 + }, + "accuracy": 0.9254642963551495, + "macro avg": { + "precision": 0.9139978207932267, + "recall": 0.8072270238762203, + "f1-score": 0.8482465360648993, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9239424880165659, + "recall": 0.9254642963551495, + "f1-score": 0.9199760030550209, + "support": 20138.0 + }, + "cross-entropy": 0.17485629796641303, + "roc-auc": 0.9582609925935802, + "score": 0.9582609925935802 + }, + "test": { + "0": { + "precision": 0.9254191144863161, + "recall": 0.9825055145660607, + "f1-score": 0.9531082826046855, + "support": 13147.0 + }, + "1": { + "precision": 0.9015832263585793, + "recall": 0.6693138500635324, + "f1-score": 0.7682771194165907, + "support": 3148.0 + }, + "accuracy": 0.9220006136851795, + "macro avg": { + "precision": 0.9135011704224477, + "recall": 0.8259096823147966, + "f1-score": 0.8606927010106381, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9208143046780242, + "recall": 0.9220006136851795, + "f1-score": 0.9174011023827694, + "support": 16295.0 + }, + "cross-entropy": 0.17807658826679365, + "roc-auc": 0.9623517001429152, + "score": 0.9623517001429152 + } + }, + "time": "0:10:44.579894", + "diversity": { + "train": { + "mean_centered_corr": 0.02127973735332489, + "mean_pairwise_disagreement": 0.024082001590469835, + "member_std": 0.02185266837477684 + }, + "val": { + "mean_centered_corr": 0.021363921463489532, + "mean_pairwise_disagreement": 0.020554144147318983, + "member_std": 0.022689195349812508 + }, + "test": { + "mean_centered_corr": 0.031094567850232124, + "mean_pairwise_disagreement": 0.026469256352136514, + "member_std": 0.02432551421225071 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..011b118b1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,280 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 4157088, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1100, + "metrics": { + "train": { + "0": { + "precision": 0.9338260534883963, + "recall": 0.9866572146111193, + "f1-score": 0.9595149597714817, + "support": 181971.0 + }, + "1": { + "precision": 0.924252823360579, + "recall": 0.6995678764551702, + "f1-score": 0.7963657379406744, + "support": 42349.0 + }, + "accuracy": 0.9324580955777461, + "macro avg": { + "precision": 0.9290394384244877, + "recall": 0.8431125455331447, + "f1-score": 0.8779403488560781, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9320187392824275, + "recall": 0.9324580955777461, + "f1-score": 0.9287142893216206, + "support": 224320.0 + }, + "cross-entropy": 0.15364890919870994, + "roc-auc": 0.9726374295788413, + "score": 0.9726374295788413 + }, + "val": { + "0": { + "precision": 0.9273736750602882, + "recall": 0.9877546144196881, + "f1-score": 0.9566122874002083, + "support": 16741.0 + }, + "1": { + "precision": 0.9111400086692675, + "recall": 0.6187812775978805, + "f1-score": 0.7370266479663394, + "support": 3397.0 + }, + "accuracy": 0.9255139537193365, + "macro avg": { + "precision": 0.9192568418647779, + "recall": 0.8032679460087844, + "f1-score": 0.8468194676832739, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9246352817376993, + "recall": 0.9255139537193365, + "f1-score": 0.91957124970248, + "support": 20138.0 + }, + "cross-entropy": 0.17514310049503773, + "roc-auc": 0.9588554974164651, + "score": 0.9588554974164651 + }, + "test": { + "0": { + "precision": 0.9220917822838848, + "recall": 0.9857762227124058, + "f1-score": 0.9528711124182045, + "support": 13147.0 + }, + "1": { + "precision": 0.9165178571428572, + "recall": 0.6521601016518425, + "f1-score": 0.7620638455827765, + "support": 3148.0 + }, + "accuracy": 0.9213255599877263, + "macro avg": { + "precision": 0.919304819713371, + "recall": 0.8189681621821241, + "f1-score": 0.8574674790004906, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9210149663069621, + "recall": 0.9213255599877263, + "f1-score": 0.9160094201200807, + "support": 16295.0 + }, + "cross-entropy": 0.17819981543536048, + "roc-auc": 0.9627966975715613, + "score": 0.9627966975715613 + } + }, + "time": "0:10:44.579894", + "diversity": { + "train": { + "mean_centered_corr": 0.02127973735332489, + "mean_pairwise_disagreement": 0.024082001590469835, + "member_std": 0.02185266837477684 + }, + "val": { + "mean_centered_corr": 0.021363921463489532, + "mean_pairwise_disagreement": 0.020554144147318983, + "member_std": 0.022689195349812508 + }, + "test": { + "mean_centered_corr": 0.031094567850232124, + "mean_pairwise_disagreement": 0.026469256352136514, + "member_std": 0.02432551421225071 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 5, + 31, + 15, + 13, + 19, + 28 + ], + "head_selection_time": "0:00:10.595939" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..d737c8b9f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 4157088, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9432888064186233, + "recall": 0.9833434997884278, + "f1-score": 0.9628997844856579, + "support": 181971.0 + }, + "1": { + "precision": 0.9124545086938941, + "recall": 0.7459680275803443, + "f1-score": 0.820854607579478, + "support": 42349.0 + }, + "accuracy": 0.9385297788873038, + "macro avg": { + "precision": 0.9278716575562587, + "recall": 0.8646557636843861, + "f1-score": 0.891877196032568, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9374676505950474, + "recall": 0.9385297788873038, + "f1-score": 0.9360833116040611, + "support": 224320.0 + }, + "cross-entropy": 0.14210167519909825, + "roc-auc": 0.9767605177962447, + "score": 0.9767605177962447 + }, + "val": { + "0": { + "precision": 0.9303139823808448, + "recall": 0.9840511319514963, + "f1-score": 0.9564283433481379, + "support": 16741.0 + }, + "1": { + "precision": 0.8901234567901235, + "recall": 0.6367382984986752, + "f1-score": 0.7424060408443452, + "support": 3397.0 + }, + "accuracy": 0.9254642963551495, + "macro avg": { + "precision": 0.9102187195854842, + "recall": 0.8103947152250858, + "f1-score": 0.8494171920962416, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.923534400722702, + "recall": 0.9254642963551495, + "f1-score": 0.9203257630717756, + "support": 20138.0 + }, + "cross-entropy": 0.1861201295625259, + "roc-auc": 0.9565663135937417, + "score": 0.9565663135937417 + }, + "test": { + "0": { + "precision": 0.9246846846846847, + "recall": 0.9758880352932228, + "f1-score": 0.9495966249722448, + "support": 13147.0 + }, + "1": { + "precision": 0.8690082644628099, + "recall": 0.6680432020330368, + "f1-score": 0.7553879310344828, + "support": 3148.0 + }, + "accuracy": 0.9164160785517029, + "macro avg": { + "precision": 0.8968464745737472, + "recall": 0.8219656186631298, + "f1-score": 0.8524922780033638, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9139286631530208, + "recall": 0.9164160785517029, + "f1-score": 0.9120778174536149, + "support": 16295.0 + }, + "cross-entropy": 0.19882303213503738, + "roc-auc": 0.9586702567362371, + "score": 0.9586702567362371 + } + }, + "time": "0:12:11.920322", + "diversity": { + "train": { + "mean_centered_corr": 0.0516890287399292, + "mean_pairwise_disagreement": 0.030828574027311213, + "member_std": 0.028233278542757034 + }, + "val": { + "mean_centered_corr": 0.061428945511579514, + "mean_pairwise_disagreement": 0.028893978323759607, + "member_std": 0.030158301815390587 + }, + "test": { + "mean_centered_corr": 0.0725783258676529, + "mean_pairwise_disagreement": 0.03697737778261687, + "member_std": 0.032935526221990585 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.906547" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..7f4852bd8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..5ffb781a1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-evaluation/0/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 4157088, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9477062667879275, + "recall": 0.9849591418412824, + "f1-score": 0.9659736725097346, + "support": 181971.0 + }, + "1": { + "precision": 0.9222354813046937, + "recall": 0.7664643793241871, + "f1-score": 0.8371655167966987, + "support": 42349.0 + }, + "accuracy": 0.9437098787446505, + "macro avg": { + "precision": 0.9349708740463105, + "recall": 0.8757117605827347, + "f1-score": 0.9015695946532167, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9428976795267405, + "recall": 0.9437098787446505, + "f1-score": 0.941656194860433, + "support": 224320.0 + }, + "cross-entropy": 0.13317658377318187, + "roc-auc": 0.9805624896867443, + "score": 0.9805624896867443 + }, + "val": { + "0": { + "precision": 0.935017639695004, + "recall": 0.9815423212472373, + "f1-score": 0.957715284860848, + "support": 16741.0 + }, + "1": { + "precision": 0.8794851794071763, + "recall": 0.6638210185457757, + "f1-score": 0.7565844656936755, + "support": 3397.0 + }, + "accuracy": 0.9279471645645049, + "macro avg": { + "precision": 0.9072514095510902, + "recall": 0.8226816698965065, + "f1-score": 0.8571498752772617, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.925650087425774, + "recall": 0.9279471645645049, + "f1-score": 0.923787318195296, + "support": 20138.0 + }, + "cross-entropy": 0.17461587348635396, + "roc-auc": 0.9580761824634811, + "score": 0.9580761824634811 + }, + "test": { + "0": { + "precision": 0.9301449275362319, + "recall": 0.9763444131741081, + "f1-score": 0.9526848999888671, + "support": 13147.0 + }, + "1": { + "precision": 0.8753507014028056, + "recall": 0.6937738246505718, + "f1-score": 0.7740563530037214, + "support": 3148.0 + }, + "accuracy": 0.9217551396133783, + "macro avg": { + "precision": 0.9027478144695187, + "recall": 0.83505911891234, + "f1-score": 0.8633706264962943, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9195593353995012, + "recall": 0.9217551396133783, + "f1-score": 0.9181759913721602, + "support": 16295.0 + }, + "cross-entropy": 0.17887228382743733, + "roc-auc": 0.962025146401907, + "score": 0.962025146401907 + } + }, + "time": "0:12:11.920322", + "diversity": { + "train": { + "mean_centered_corr": 0.0516890287399292, + "mean_pairwise_disagreement": 0.030828574027311213, + "member_std": 0.028233278542757034 + }, + "val": { + "mean_centered_corr": 0.061428945511579514, + "mean_pairwise_disagreement": 0.028893978323759607, + "member_std": 0.030158301815390587 + }, + "test": { + "mean_centered_corr": 0.0725783258676529, + "mean_pairwise_disagreement": 0.03697737778261687, + "member_std": 0.032935526221990585 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3c4449fab --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,282 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 4157088, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9449847896337933, + "recall": 0.9866901868979123, + "f1-score": 0.9653872727321606, + "support": 181971.0 + }, + "1": { + "precision": 0.9294247916545253, + "recall": 0.7531700866608421, + "f1-score": 0.8320659475393585, + "support": 42349.0 + }, + "accuracy": 0.9426043152639088, + "macro avg": { + "precision": 0.9372047906441593, + "recall": 0.8699301367793772, + "f1-score": 0.8987266101357596, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9420472434746276, + "recall": 0.9426043152639088, + "f1-score": 0.9402177613172623, + "support": 224320.0 + }, + "cross-entropy": 0.13407185107135744, + "roc-auc": 0.9801324950911569, + "score": 0.9801324950911569 + }, + "val": { + "0": { + "precision": 0.9316949152542373, + "recall": 0.9850666029508393, + "f1-score": 0.9576376992537964, + "support": 16741.0 + }, + "1": { + "precision": 0.8974569319114027, + "recall": 0.644097733294083, + "f1-score": 0.7499571550985433, + "support": 3397.0 + }, + "accuracy": 0.927549905651008, + "macro avg": { + "precision": 0.91457592358282, + "recall": 0.8145821681224612, + "f1-score": 0.8537974271761699, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9259194445314441, + "recall": 0.927549905651008, + "f1-score": 0.9226048852456827, + "support": 20138.0 + }, + "cross-entropy": 0.1751815095057731, + "roc-auc": 0.9588770556676071, + "score": 0.9588770556676071 + }, + "test": { + "0": { + "precision": 0.926901215914814, + "recall": 0.9799193732410436, + "f1-score": 0.9526732233971752, + "support": 13147.0 + }, + "1": { + "precision": 0.8898163606010017, + "recall": 0.6772554002541296, + "f1-score": 0.7691197691197691, + "support": 3148.0 + }, + "accuracy": 0.9214482970236268, + "macro avg": { + "precision": 0.9083587882579078, + "recall": 0.8285873867475866, + "f1-score": 0.8608964962584722, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9197368633816517, + "recall": 0.9214482970236268, + "f1-score": 0.9172128813250504, + "support": 16295.0 + }, + "cross-entropy": 0.180176006712123, + "roc-auc": 0.9621469244895637, + "score": 0.9621469244895637 + } + }, + "time": "0:12:11.920322", + "diversity": { + "train": { + "mean_centered_corr": 0.0516890287399292, + "mean_pairwise_disagreement": 0.030828574027311213, + "member_std": 0.028233278542757034 + }, + "val": { + "mean_centered_corr": 0.061428945511579514, + "mean_pairwise_disagreement": 0.028893978323759607, + "member_std": 0.030158301815390587 + }, + "test": { + "mean_centered_corr": 0.0725783258676529, + "mean_pairwise_disagreement": 0.03697737778261687, + "member_std": 0.032935526221990585 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 24, + 31, + 12, + 8, + 30, + 24, + 9 + ], + "head_selection_time": "0:00:09.478138" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..db2179a27 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,275 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 4157088, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9402544726609582, + "recall": 0.9860087596375247, + "f1-score": 0.9625882182528293, + "support": 181971.0 + }, + "1": { + "precision": 0.9239863856213053, + "recall": 0.7307846702401474, + "f1-score": 0.816106957794391, + "support": 42349.0 + }, + "accuracy": 0.9378254279600571, + "macro avg": { + "precision": 0.9321204291411318, + "recall": 0.8583967149388361, + "f1-score": 0.8893475880236101, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.937183247544864, + "recall": 0.9378254279600571, + "f1-score": 0.9349342645297801, + "support": 224320.0 + }, + "cross-entropy": 0.14254385707563483, + "roc-auc": 0.9770372580348661, + "score": 0.9770372580348661 + }, + "val": { + "0": { + "precision": 0.9269017321598745, + "recall": 0.9876948808314916, + "f1-score": 0.9563331405436668, + "support": 16741.0 + }, + "1": { + "precision": 0.9103958242714224, + "recall": 0.6161318810715337, + "f1-score": 0.7349016853932584, + "support": 3397.0 + }, + "accuracy": 0.9250173800774655, + "macro avg": { + "precision": 0.9186487782156485, + "recall": 0.8019133809515127, + "f1-score": 0.8456174129684626, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9241174154900427, + "recall": 0.9250173800774655, + "f1-score": 0.9189807394538894, + "support": 20138.0 + }, + "cross-entropy": 0.19012849944818658, + "roc-auc": 0.9560226359526883, + "score": 0.9560226359526883 + }, + "test": { + "0": { + "precision": 0.9207786651454649, + "recall": 0.9822012626454705, + "f1-score": 0.9504986934599389, + "support": 13147.0 + }, + "1": { + "precision": 0.8969616908850726, + "recall": 0.6470775095298602, + "f1-score": 0.7517992249492527, + "support": 3148.0 + }, + "accuracy": 0.917459343356858, + "macro avg": { + "precision": 0.9088701780152688, + "recall": 0.8146393860876653, + "f1-score": 0.8511489592045958, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9161775092711651, + "recall": 0.917459343356858, + "f1-score": 0.9121123217587029, + "support": 16295.0 + }, + "cross-entropy": 0.2007697564863984, + "roc-auc": 0.9594298354768371, + "score": 0.9594298354768371 + } + }, + "time": "0:12:13.002733", + "diversity": { + "train": { + "mean_centered_corr": 0.05281975865364075, + "mean_pairwise_disagreement": 0.030102508728889605, + "member_std": 0.02849932573735714 + }, + "val": { + "mean_centered_corr": 0.059580858796834946, + "mean_pairwise_disagreement": 0.027920053245509214, + "member_std": 0.030231283977627754 + }, + "test": { + "mean_centered_corr": 0.07347472012042999, + "mean_pairwise_disagreement": 0.03518185372516802, + "member_std": 0.03243069350719452 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.973893" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..949a0af96 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/homesite-insurance" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0018580623030886075 +weight_decay = 0.0001614529849348179 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 704 +dropout = 0.0 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 8 + +[bins] +n_bins = 15 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..40cdb12ad --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-evaluation/0/report.json @@ -0,0 +1,271 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 4157088, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9450154648936394, + "recall": 0.9872726972979211, + "f1-score": 0.9656820191410964, + "support": 181971.0 + }, + "1": { + "precision": 0.9323044545773412, + "recall": 0.7531700866608421, + "f1-score": 0.8332179569232376, + "support": 42349.0 + }, + "accuracy": 0.9430768544935806, + "macro avg": { + "precision": 0.9386599597354903, + "recall": 0.8702213919793815, + "f1-score": 0.899449988032167, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9426157743806003, + "recall": 0.9430768544935806, + "f1-score": 0.9406743489785425, + "support": 224320.0 + }, + "cross-entropy": 0.1326793673321411, + "roc-auc": 0.9809027579015627, + "score": 0.9809027579015627 + }, + "val": { + "0": { + "precision": 0.9323176257362936, + "recall": 0.98327459530494, + "f1-score": 0.9571183533447685, + "support": 16741.0 + }, + "1": { + "precision": 0.887187751813054, + "recall": 0.6482190167795113, + "f1-score": 0.7491069909848613, + "support": 3397.0 + }, + "accuracy": 0.9267553878240143, + "macro avg": { + "precision": 0.9097526887746739, + "recall": 0.8157468060422257, + "f1-score": 0.8531126721648149, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.9247048448386253, + "recall": 0.9267553878240143, + "f1-score": 0.9220297349151029, + "support": 20138.0 + }, + "cross-entropy": 0.17484437515862924, + "roc-auc": 0.958187182838957, + "score": 0.958187182838957 + }, + "test": { + "0": { + "precision": 0.927964270278058, + "recall": 0.979843310260896, + "f1-score": 0.9531984165155943, + "support": 13147.0 + }, + "1": { + "precision": 0.8901782014090344, + "recall": 0.6823379923761118, + "f1-score": 0.7725229275310196, + "support": 3148.0 + }, + "accuracy": 0.9223688247928813, + "macro avg": { + "precision": 0.9090712358435462, + "recall": 0.8310906513185039, + "f1-score": 0.862860672023307, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9206644516343214, + "recall": 0.9223688247928813, + "f1-score": 0.9182940630744504, + "support": 16295.0 + }, + "cross-entropy": 0.17826501548417395, + "roc-auc": 0.9624786972914716, + "score": 0.9624786972914716 + } + }, + "time": "0:12:13.002733", + "diversity": { + "train": { + "mean_centered_corr": 0.05281975865364075, + "mean_pairwise_disagreement": 0.030102508728889605, + "member_std": 0.02849932573735714 + }, + "val": { + "mean_centered_corr": 0.059580858796834946, + "mean_pairwise_disagreement": 0.027920053245509214, + "member_std": 0.030231283977627754 + }, + "test": { + "mean_centered_corr": 0.07347472012042999, + "mean_pairwise_disagreement": 0.03518185372516802, + "member_std": 0.03243069350719452 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8c051bd05 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/homesite-insurance/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,287 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/homesite-insurance", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0018580623030886075, + "weight_decay": 0.0001614529849348179 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 704, + "dropout": 0.0 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 8 + } + }, + "bins": { + "n_bins": 15 + } + }, + "n_parameters": 4157088, + "prediction_type": "probs", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 220, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 2206, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 704 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 704, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 704 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1760, + "metrics": { + "train": { + "0": { + "precision": 0.9441635038952028, + "recall": 0.9870363959092383, + "f1-score": 0.965124057978802, + "support": 181971.0 + }, + "1": { + "precision": 0.9307927008155841, + "recall": 0.7491794375309925, + "f1-score": 0.8301694250016354, + "support": 42349.0 + }, + "accuracy": 0.9421317760342368, + "macro avg": { + "precision": 0.9374781023553934, + "recall": 0.8681079167201153, + "f1-score": 0.8976467414902187, + "support": 224320.0 + }, + "weighted avg": { + "precision": 0.9416392522028937, + "recall": 0.9421317760342368, + "f1-score": 0.9396461971017066, + "support": 224320.0 + }, + "cross-entropy": 0.13438484231859613, + "roc-auc": 0.9800307814893711, + "score": 0.9800307814893711 + }, + "val": { + "0": { + "precision": 0.931937765205092, + "recall": 0.9839316647751031, + "f1-score": 0.9572291957229195, + "support": 16741.0 + }, + "1": { + "precision": 0.8907835972391392, + "recall": 0.6458639976449808, + "f1-score": 0.7488054607508532, + "support": 3397.0 + }, + "accuracy": 0.9269043599165756, + "macro avg": { + "precision": 0.9113606812221156, + "recall": 0.8148978312100419, + "f1-score": 0.8530173282368864, + "support": 20138.0 + }, + "weighted avg": { + "precision": 0.924995630505502, + "recall": 0.9269043599165756, + "f1-score": 0.9220710157795234, + "support": 20138.0 + }, + "cross-entropy": 0.17479880731904499, + "roc-auc": 0.9588073342436448, + "score": 0.9588073342436448 + }, + "test": { + "0": { + "precision": 0.9273630408177957, + "recall": 0.979843310260896, + "f1-score": 0.952881130261114, + "support": 13147.0 + }, + "1": { + "precision": 0.8897670549084858, + "recall": 0.6794790343074968, + "f1-score": 0.7705331412103746, + "support": 3148.0 + }, + "accuracy": 0.9218165081313286, + "macro avg": { + "precision": 0.9085650478631409, + "recall": 0.8296611722841964, + "f1-score": 0.8617071357357443, + "support": 16295.0 + }, + "weighted avg": { + "precision": 0.9200999439388448, + "recall": 0.9218165081313286, + "f1-score": 0.917653669719124, + "support": 16295.0 + }, + "cross-entropy": 0.17874164813587776, + "roc-auc": 0.9625660923992208, + "score": 0.9625660923992208 + } + }, + "time": "0:12:13.002733", + "diversity": { + "train": { + "mean_centered_corr": 0.05281975865364075, + "mean_pairwise_disagreement": 0.030102508728889605, + "member_std": 0.02849932573735714 + }, + "val": { + "mean_centered_corr": 0.059580858796834946, + "mean_pairwise_disagreement": 0.027920053245509214, + "member_std": 0.030231283977627754 + }, + "test": { + "mean_centered_corr": 0.07347472012042999, + "mean_pairwise_disagreement": 0.03518185372516802, + "member_std": 0.03243069350719452 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 31, + 24, + 30, + 6, + 7, + 0, + 9, + 2, + 30, + 24, + 31, + 11 + ], + "head_selection_time": "0:00:13.085238" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f5cc9f8c2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-best-head-evaluation/0/report.json @@ -0,0 +1,702 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.238513825213292, + "mae": 0.12740404903888702, + "r2": 0.6365530490875244, + "score": -0.238513825213292 + }, + "val": { + "rmse": 0.2376718937442043, + "mae": 0.13975045084953308, + "r2": 0.6226825714111328, + "score": -0.2376718937442043 + }, + "test": { + "rmse": 0.2539783962159738, + "mae": 0.17701080441474915, + "r2": 0.5592085719108582, + "score": -0.2539783962159738 + } + }, + "time": "0:03:25.619596", + "diversity": { + "train": { + "mean_centered_corr": -0.018900908529758453, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04601670801639557 + }, + "val": { + "mean_centered_corr": -0.01799248345196247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05731586739420891 + }, + "test": { + "mean_centered_corr": -0.019703488796949387, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07458258420228958 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015338" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0.toml new file mode 100644 index 000000000..aaa491b98 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0.toml @@ -0,0 +1,75 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0/report.json new file mode 100644 index 000000000..624449948 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2331321585964892, + "mae": 0.12105797231197357, + "r2": 0.6527692079544067, + "score": -0.2331321585964892 + }, + "val": { + "rmse": 0.2348792849855834, + "mae": 0.12763135135173798, + "r2": 0.6314972639083862, + "score": -0.2348792849855834 + }, + "test": { + "rmse": 0.2326272249658597, + "mae": 0.1447165459394455, + "r2": 0.6302052736282349, + "score": -0.2326272249658597 + } + }, + "time": "0:03:25.619596", + "diversity": { + "train": { + "mean_centered_corr": -0.018900908529758453, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04601670801639557 + }, + "val": { + "mean_centered_corr": -0.01799248345196247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05731586739420891 + }, + "test": { + "mean_centered_corr": -0.019703488796949387, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07458258420228958 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..94589782c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-greedy-heads-evaluation/0/report.json @@ -0,0 +1,710 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_four_combined-evaluation/_esam_diagnostics/all_four_combined-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23288428364808966, + "mae": 0.12141954898834229, + "r2": 0.6535071730613708, + "score": -0.23288428364808966 + }, + "val": { + "rmse": 0.23335850745168515, + "mae": 0.1262979656457901, + "r2": 0.6362537145614624, + "score": -0.23335850745168515 + }, + "test": { + "rmse": 0.23263890687901315, + "mae": 0.14496733248233795, + "r2": 0.6301681995391846, + "score": -0.23263890687901315 + } + }, + "time": "0:03:25.619596", + "diversity": { + "train": { + "mean_centered_corr": -0.018900908529758453, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04601670801639557 + }, + "val": { + "mean_centered_corr": -0.01799248345196247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05731586739420891 + }, + "test": { + "mean_centered_corr": -0.019703488796949387, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07458258420228958 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 10, + 19, + 8, + 12, + 28, + 21, + 24, + 20 + ], + "head_selection_time": "0:00:00.147069" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9c84f2134 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,249 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23669818404848408, + "mae": 0.1265603005886078, + "r2": 0.6420653462409973, + "score": -0.23669818404848408 + }, + "val": { + "rmse": 0.23808479027162685, + "mae": 0.13432005047798157, + "r2": 0.6213704347610474, + "score": -0.23808479027162685 + }, + "test": { + "rmse": 0.27950919024280946, + "mae": 0.20337718725204468, + "r2": 0.46613460779190063, + "score": -0.27950919024280946 + } + }, + "time": "0:02:54.620098", + "diversity": { + "train": { + "mean_centered_corr": -0.018507609143853188, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046238355338573456 + }, + "val": { + "mean_centered_corr": -0.019183674827218056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059328801929950714 + }, + "test": { + "mean_centered_corr": -0.020014170557260513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07509658485651016 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015242" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..f1eb43eee --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0.toml @@ -0,0 +1,64 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..b1b517b94 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/0/report.json @@ -0,0 +1,245 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2320688756626341, + "mae": 0.1213240846991539, + "r2": 0.6559293270111084, + "score": -0.2320688756626341 + }, + "val": { + "rmse": 0.23469635866718716, + "mae": 0.12645497918128967, + "r2": 0.6320710182189941, + "score": -0.23469635866718716 + }, + "test": { + "rmse": 0.23237236681744675, + "mae": 0.14428266882896423, + "r2": 0.6310151219367981, + "score": -0.23237236681744675 + } + }, + "time": "0:02:54.620098", + "diversity": { + "train": { + "mean_centered_corr": -0.018507609143853188, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046238355338573456 + }, + "val": { + "mean_centered_corr": -0.019183674827218056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059328801929950714 + }, + "test": { + "mean_centered_corr": -0.020014170557260513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07509658485651016 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1120f9b75 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,256 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_cf_fisd-evaluation/_esam_diagnostics/all_minus_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23281312116757674, + "mae": 0.12442371994256973, + "r2": 0.653718888759613, + "score": -0.23281312116757674 + }, + "val": { + "rmse": 0.23321538793887137, + "mae": 0.12674209475517273, + "r2": 0.6366997957229614, + "score": -0.23321538793887137 + }, + "test": { + "rmse": 0.23519795893846665, + "mae": 0.1507532298564911, + "r2": 0.6219870448112488, + "score": -0.23519795893846665 + } + }, + "time": "0:02:54.620098", + "diversity": { + "train": { + "mean_centered_corr": -0.018507609143853188, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046238355338573456 + }, + "val": { + "mean_centered_corr": -0.019183674827218056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059328801929950714 + }, + "test": { + "mean_centered_corr": -0.020014170557260513, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07509658485651016 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 24, + 19, + 4, + 12, + 24, + 31, + 18 + ], + "head_selection_time": "0:00:00.130392" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f98d1d253 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-best-head-evaluation/0/report.json @@ -0,0 +1,692 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23559077756802618, + "mae": 0.12391819804906845, + "r2": 0.6454067230224609, + "score": -0.23559077756802618 + }, + "val": { + "rmse": 0.23762339316571812, + "mae": 0.13952432572841644, + "r2": 0.6228364706039429, + "score": -0.23762339316571812 + }, + "test": { + "rmse": 0.2548487995005296, + "mae": 0.1803659200668335, + "r2": 0.5561821460723877, + "score": -0.2548487995005296 + } + }, + "time": "0:02:05.501890", + "diversity": { + "train": { + "mean_centered_corr": -0.019058862701058388, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04700104147195816 + }, + "val": { + "mean_centered_corr": -0.01848544180393219, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06533127278089523 + }, + "test": { + "mean_centered_corr": -0.01928732916712761, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08419054746627808 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015176" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0.toml new file mode 100644 index 000000000..1ee60827a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0/report.json new file mode 100644 index 000000000..f5a72f4b6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23240700041031828, + "mae": 0.11997766047716141, + "r2": 0.6549259424209595, + "score": -0.23240700041031828 + }, + "val": { + "rmse": 0.23494022877441162, + "mae": 0.1266379952430725, + "r2": 0.6313060522079468, + "score": -0.23494022877441162 + }, + "test": { + "rmse": 0.2319230328213383, + "mae": 0.14268861711025238, + "r2": 0.6324407458305359, + "score": -0.2319230328213383 + } + }, + "time": "0:02:05.501890", + "diversity": { + "train": { + "mean_centered_corr": -0.019058862701058388, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04700104147195816 + }, + "val": { + "mean_centered_corr": -0.01848544180393219, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06533127278089523 + }, + "test": { + "mean_centered_corr": -0.01928732916712761, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08419054746627808 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4150daf8f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_esam-greedy-heads-evaluation/0/report.json @@ -0,0 +1,697 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23248251756958926, + "mae": 0.12231885641813278, + "r2": 0.6547016501426697, + "score": -0.23248251756958926 + }, + "val": { + "rmse": 0.23366970594362974, + "mae": 0.1270563155412674, + "r2": 0.6352829337120056, + "score": -0.23366970594362974 + }, + "test": { + "rmse": 0.23132655964186158, + "mae": 0.14191654324531555, + "r2": 0.6343289613723755, + "score": -0.23132655964186158 + } + }, + "time": "0:02:05.501890", + "diversity": { + "train": { + "mean_centered_corr": -0.019058862701058388, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04700104147195816 + }, + "val": { + "mean_centered_corr": -0.01848544180393219, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06533127278089523 + }, + "test": { + "mean_centered_corr": -0.01928732916712761, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08419054746627808 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25, + 21, + 19, + 0, + 22, + 3 + ], + "head_selection_time": "0:00:00.101481" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5dabb8889 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,666 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23953673006222745, + "mae": 0.1305936723947525, + "r2": 0.6334289908409119, + "score": -0.23953673006222745 + }, + "val": { + "rmse": 0.23769862439897374, + "mae": 0.13297532498836517, + "r2": 0.6225976347923279, + "score": -0.23769862439897374 + }, + "test": { + "rmse": 0.24291600363173604, + "mae": 0.1576826125383377, + "r2": 0.5967708826065063, + "score": -0.24291600363173604 + } + }, + "time": "0:03:16.948884", + "diversity": { + "train": { + "mean_centered_corr": -0.02016131393611431, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04428550973534584 + }, + "val": { + "mean_centered_corr": -0.01863192766904831, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06180477887392044 + }, + "test": { + "mean_centered_corr": -0.011988693848252296, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08672565221786499 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015532" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0.toml new file mode 100644 index 000000000..d4514eeae --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0.toml @@ -0,0 +1,62 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0/report.json new file mode 100644 index 000000000..8fdfb8b08 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/0/report.json @@ -0,0 +1,662 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23349763844998986, + "mae": 0.12047048658132553, + "r2": 0.6516796350479126, + "score": -0.23349763844998986 + }, + "val": { + "rmse": 0.23539867958344296, + "mae": 0.12546104192733765, + "r2": 0.6298657655715942, + "score": -0.23539867958344296 + }, + "test": { + "rmse": 0.2332565482846656, + "mae": 0.1435696929693222, + "r2": 0.6282018423080444, + "score": -0.2332565482846656 + } + }, + "time": "0:03:16.948884", + "diversity": { + "train": { + "mean_centered_corr": -0.02016131393611431, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04428550973534584 + }, + "val": { + "mean_centered_corr": -0.01863192766904831, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06180477887392044 + }, + "test": { + "mean_centered_corr": -0.011988693848252296, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08672565221786499 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3b701aadc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,673 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_mfb-evaluation/_esam_diagnostics/all_minus_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2332399859921054, + "mae": 0.12097182869911194, + "r2": 0.6524479389190674, + "score": -0.2332399859921054 + }, + "val": { + "rmse": 0.23349088968550719, + "mae": 0.12680040299892426, + "r2": 0.635840892791748, + "score": -0.23349088968550719 + }, + "test": { + "rmse": 0.23959467004131907, + "mae": 0.15605996549129486, + "r2": 0.6077220439910889, + "score": -0.23959467004131907 + } + }, + "time": "0:03:16.948884", + "diversity": { + "train": { + "mean_centered_corr": -0.02016131393611431, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04428550973534584 + }, + "val": { + "mean_centered_corr": -0.01863192766904831, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06180477887392044 + }, + "test": { + "mean_centered_corr": -0.011988693848252296, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08672565221786499 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 12, + 28, + 16, + 19, + 25, + 10, + 20 + ], + "head_selection_time": "0:00:00.131053" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-best-head-evaluation/0/report.json new file mode 100644 index 000000000..fc5abdc56 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-best-head-evaluation/0/report.json @@ -0,0 +1,689 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2385659861695932, + "mae": 0.12279489636421204, + "r2": 0.6363940834999084, + "score": -0.2385659861695932 + }, + "val": { + "rmse": 0.2383111872393019, + "mae": 0.1314251869916916, + "r2": 0.6206499934196472, + "score": -0.2383111872393019 + }, + "test": { + "rmse": 0.2360172512727007, + "mae": 0.14784888923168182, + "r2": 0.6193488836288452, + "score": -0.2360172512727007 + } + }, + "time": "0:02:10.492471", + "diversity": { + "train": { + "mean_centered_corr": -0.019148705527186394, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04757876694202423 + }, + "val": { + "mean_centered_corr": -0.01928521692752838, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06040926277637482 + }, + "test": { + "mean_centered_corr": -0.019908146932721138, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07882117480039597 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28 + ], + "head_selection_time": "0:00:00.015224" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0.toml new file mode 100644 index 000000000..d5a6282f3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0.toml @@ -0,0 +1,68 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0/report.json new file mode 100644 index 000000000..e3195bd55 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/0/report.json @@ -0,0 +1,685 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23440638967758307, + "mae": 0.12211208790540695, + "r2": 0.6489630937576294, + "score": -0.23440638967758307 + }, + "val": { + "rmse": 0.23564260539703746, + "mae": 0.12916921079158783, + "r2": 0.6290982365608215, + "score": -0.23564260539703746 + }, + "test": { + "rmse": 0.2337545527178023, + "mae": 0.14542868733406067, + "r2": 0.6266125440597534, + "score": -0.2337545527178023 + } + }, + "time": "0:02:10.492471", + "diversity": { + "train": { + "mean_centered_corr": -0.019148705527186394, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04757876694202423 + }, + "val": { + "mean_centered_corr": -0.01928521692752838, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06040926277637482 + }, + "test": { + "mean_centered_corr": -0.019908146932721138, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07882117480039597 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..732541603 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-greedy-heads-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/all_minus_rla-evaluation/_esam_diagnostics/all_minus_rla-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2332182711566056, + "mae": 0.1229761615395546, + "r2": 0.6525126695632935, + "score": -0.2332182711566056 + }, + "val": { + "rmse": 0.23419188657775022, + "mae": 0.1297776848077774, + "r2": 0.6336510181427002, + "score": -0.23419188657775022 + }, + "test": { + "rmse": 0.23442550750747457, + "mae": 0.14977632462978363, + "r2": 0.6244659423828125, + "score": -0.23442550750747457 + } + }, + "time": "0:02:10.492471", + "diversity": { + "train": { + "mean_centered_corr": -0.019148705527186394, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04757876694202423 + }, + "val": { + "mean_centered_corr": -0.01928521692752838, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06040926277637482 + }, + "test": { + "mean_centered_corr": -0.019908146932721138, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07882117480039597 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28, + 19, + 25, + 12, + 24, + 22 + ], + "head_selection_time": "0:00:00.101853" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5c112e278 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-best-head-evaluation/0/report.json @@ -0,0 +1,643 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23931750895741843, + "mae": 0.13275523483753204, + "r2": 0.6340996623039246, + "score": -0.23931750895741843 + }, + "val": { + "rmse": 0.2394238965987828, + "mae": 0.13270841538906097, + "r2": 0.6170992255210876, + "score": -0.2394238965987828 + }, + "test": { + "rmse": 0.23525657161407812, + "mae": 0.14720264077186584, + "r2": 0.6217986345291138, + "score": -0.23525657161407812 + } + }, + "time": "0:01:03.360837", + "diversity": { + "train": { + "mean_centered_corr": -0.019119493663311005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046866077929735184 + }, + "val": { + "mean_centered_corr": -0.02003737911581993, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058194298297166824 + }, + "test": { + "mean_centered_corr": -0.01959315314888954, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07949266582727432 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.015297" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0.toml new file mode 100644 index 000000000..2692a646a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0.toml @@ -0,0 +1,45 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.05 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0/report.json new file mode 100644 index 000000000..40a4a148f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-evaluation/0/report.json @@ -0,0 +1,639 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23390890506009623, + "mae": 0.12171443551778793, + "r2": 0.6504515409469604, + "score": -0.23390890506009623 + }, + "val": { + "rmse": 0.2359593327633197, + "mae": 0.12757939100265503, + "r2": 0.6281005144119263, + "score": -0.2359593327633197 + }, + "test": { + "rmse": 0.23536697907032925, + "mae": 0.1493176817893982, + "r2": 0.621443510055542, + "score": -0.23536697907032925 + } + }, + "time": "0:01:03.360837", + "diversity": { + "train": { + "mean_centered_corr": -0.019119493663311005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046866077929735184 + }, + "val": { + "mean_centered_corr": -0.02003737911581993, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058194298297166824 + }, + "test": { + "mean_centered_corr": -0.01959315314888954, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07949266582727432 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ffd05ad90 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,647 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.05, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23536915535287303, + "mae": 0.12580418586730957, + "r2": 0.646073579788208, + "score": -0.23536915535287303 + }, + "val": { + "rmse": 0.23452905519184072, + "mae": 0.1281842142343521, + "r2": 0.632595419883728, + "score": -0.23452905519184072 + }, + "test": { + "rmse": 0.23270925018080982, + "mae": 0.14333613216876984, + "r2": 0.6299444437026978, + "score": -0.23270925018080982 + } + }, + "time": "0:01:03.360837", + "diversity": { + "train": { + "mean_centered_corr": -0.019119493663311005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046866077929735184 + }, + "val": { + "mean_centered_corr": -0.02003737911581993, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058194298297166824 + }, + "test": { + "mean_centered_corr": -0.01959315314888954, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07949266582727432 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24, + 16, + 0, + 20, + 29 + ], + "head_selection_time": "0:00:00.087503" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a34d94223 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-best-head-evaluation/0/report.json @@ -0,0 +1,643 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24287177140617242, + "mae": 0.12792739272117615, + "r2": 0.6231504678726196, + "score": -0.24287177140617242 + }, + "val": { + "rmse": 0.2388197411165918, + "mae": 0.13359415531158447, + "r2": 0.61902916431427, + "score": -0.2388197411165918 + }, + "test": { + "rmse": 0.2462523116085554, + "mae": 0.16663780808448792, + "r2": 0.5856186151504517, + "score": -0.2462523116085554 + } + }, + "time": "0:01:04.483422", + "diversity": { + "train": { + "mean_centered_corr": -0.018840299919247627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04810040444135666 + }, + "val": { + "mean_centered_corr": -0.020352300256490707, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061659909784793854 + }, + "test": { + "mean_centered_corr": -0.019725581631064415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08170976489782333 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015081" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0.toml new file mode 100644 index 000000000..523756861 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0.toml @@ -0,0 +1,45 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0/report.json new file mode 100644 index 000000000..ad9df262e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-evaluation/0/report.json @@ -0,0 +1,639 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23279361488245573, + "mae": 0.12178773432970047, + "r2": 0.6537768840789795, + "score": -0.23279361488245573 + }, + "val": { + "rmse": 0.23523413233394516, + "mae": 0.12663741409778595, + "r2": 0.6303830146789551, + "score": -0.23523413233394516 + }, + "test": { + "rmse": 0.235149605552611, + "mae": 0.14915932714939117, + "r2": 0.6221424341201782, + "score": -0.235149605552611 + } + }, + "time": "0:01:04.483422", + "diversity": { + "train": { + "mean_centered_corr": -0.018840299919247627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04810040444135666 + }, + "val": { + "mean_centered_corr": -0.020352300256490707, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061659909784793854 + }, + "test": { + "mean_centered_corr": -0.019725581631064415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08170976489782333 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0a6b873ae --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.1-greedy-heads-evaluation/0/report.json @@ -0,0 +1,649 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23477394006764257, + "mae": 0.1248951405286789, + "r2": 0.6478613615036011, + "score": -0.23477394006764257 + }, + "val": { + "rmse": 0.23327114512588593, + "mae": 0.12512965500354767, + "r2": 0.6365260481834412, + "score": -0.23327114512588593 + }, + "test": { + "rmse": 0.23367141178994547, + "mae": 0.14663751423358917, + "r2": 0.6268780827522278, + "score": -0.23367141178994547 + } + }, + "time": "0:01:04.483422", + "diversity": { + "train": { + "mean_centered_corr": -0.018840299919247627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04810040444135666 + }, + "val": { + "mean_centered_corr": -0.020352300256490707, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061659909784793854 + }, + "test": { + "mean_centered_corr": -0.019725581631064415, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08170976489782333 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 21, + 25, + 19, + 0, + 28 + ], + "head_selection_time": "0:00:00.117459" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3af7afec9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-best-head-evaluation/0/report.json @@ -0,0 +1,643 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23290555777498698, + "mae": 0.12613320350646973, + "r2": 0.6534438729286194, + "score": -0.23290555777498698 + }, + "val": { + "rmse": 0.23866861156973368, + "mae": 0.13892269134521484, + "r2": 0.6195112466812134, + "score": -0.23866861156973368 + }, + "test": { + "rmse": 0.2497089599248454, + "mae": 0.17276842892169952, + "r2": 0.5739036202430725, + "score": -0.2497089599248454 + } + }, + "time": "0:01:08.920450", + "diversity": { + "train": { + "mean_centered_corr": -0.0169442780315876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04900464415550232 + }, + "val": { + "mean_centered_corr": -0.02006734535098076, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06669005006551743 + }, + "test": { + "mean_centered_corr": -0.01514353696256876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09919938445091248 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.015245" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0.toml new file mode 100644 index 000000000..3912eb390 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0.toml @@ -0,0 +1,45 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.2 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0/report.json new file mode 100644 index 000000000..3e7f198e0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-evaluation/0/report.json @@ -0,0 +1,639 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2242219236860395, + "mae": 0.11819882690906525, + "r2": 0.6788040399551392, + "score": -0.2242219236860395 + }, + "val": { + "rmse": 0.23533942163580152, + "mae": 0.12796977162361145, + "r2": 0.6300520896911621, + "score": -0.23533942163580152 + }, + "test": { + "rmse": 0.24087782495581203, + "mae": 0.1581418365240097, + "r2": 0.6035090684890747, + "score": -0.24087782495581203 + } + }, + "time": "0:01:08.920450", + "diversity": { + "train": { + "mean_centered_corr": -0.0169442780315876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04900464415550232 + }, + "val": { + "mean_centered_corr": -0.02006734535098076, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06669005006551743 + }, + "test": { + "mean_centered_corr": -0.01514353696256876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09919938445091248 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f2d7e30ac --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/cf_fisd_lambda0.2-greedy-heads-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.2, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22441248349529827, + "mae": 0.1177876740694046, + "r2": 0.6782578825950623, + "score": -0.22441248349529827 + }, + "val": { + "rmse": 0.23337569983908155, + "mae": 0.12688370048999786, + "r2": 0.6362001895904541, + "score": -0.23337569983908155 + }, + "test": { + "rmse": 0.244512625524316, + "mae": 0.16398777067661285, + "r2": 0.5914528369903564, + "score": -0.244512625524316 + } + }, + "time": "0:01:08.920450", + "diversity": { + "train": { + "mean_centered_corr": -0.0169442780315876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04900464415550232 + }, + "val": { + "mean_centered_corr": -0.02006734535098076, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06669005006551743 + }, + "test": { + "mean_centered_corr": -0.01514353696256876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09919938445091248 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 19, + 25, + 21, + 29, + 18, + 8, + 1, + 19, + 25, + 12, + 4 + ], + "head_selection_time": "0:00:00.190618" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..fd0703522 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,653 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1406, + "metrics": { + "train": { + "rmse": 0.21463492388268743, + "mae": 0.11984407156705856, + "r2": 0.7056834697723389, + "score": -0.21463492388268743 + }, + "val": { + "rmse": 0.2404271956933659, + "mae": 0.14037233591079712, + "r2": 0.6138834357261658, + "score": -0.2404271956933659 + }, + "test": { + "rmse": 0.25751796428274565, + "mae": 0.17290249466896057, + "r2": 0.5468367338180542, + "score": -0.25751796428274565 + } + }, + "time": "0:02:42.897265", + "diversity": { + "train": { + "mean_centered_corr": -0.017316685989499092, + "mean_pairwise_disagreement": NaN, + "member_std": 0.050752341747283936 + }, + "val": { + "mean_centered_corr": -0.014526012353599072, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017978072166443 + }, + "test": { + "mean_centered_corr": -0.008432705886662006, + "mean_pairwise_disagreement": NaN, + "member_std": 0.13299168646335602 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27 + ], + "head_selection_time": "0:00:00.015175" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..690d98000 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0.toml @@ -0,0 +1,55 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..fc2c6d417 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/0/report.json @@ -0,0 +1,649 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1406, + "metrics": { + "train": { + "rmse": 0.20151074580440687, + "mae": 0.11064346134662628, + "r2": 0.740575909614563, + "score": -0.20151074580440687 + }, + "val": { + "rmse": 0.2345315410452097, + "mae": 0.12432665377855301, + "r2": 0.6325876712799072, + "score": -0.2345315410452097 + }, + "test": { + "rmse": 0.249503671176673, + "mae": 0.16559600830078125, + "r2": 0.5746039152145386, + "score": -0.249503671176673 + } + }, + "time": "0:02:42.897265", + "diversity": { + "train": { + "mean_centered_corr": -0.017316685989499092, + "mean_pairwise_disagreement": NaN, + "member_std": 0.050752341747283936 + }, + "val": { + "mean_centered_corr": -0.014526012353599072, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017978072166443 + }, + "test": { + "mean_centered_corr": -0.008432705886662006, + "mean_pairwise_disagreement": NaN, + "member_std": 0.13299168646335602 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ec09e51d6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,662 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_cf_fisd-evaluation/_esam_diagnostics/esam_cf_fisd-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1406, + "metrics": { + "train": { + "rmse": 0.20170394200431338, + "mae": 0.10968871414661407, + "r2": 0.7400782108306885, + "score": -0.20170394200431338 + }, + "val": { + "rmse": 0.23329227219621285, + "mae": 0.12547366321086884, + "r2": 0.6364601850509644, + "score": -0.23329227219621285 + }, + "test": { + "rmse": 0.23856120001818734, + "mae": 0.15210896730422974, + "r2": 0.6110988855361938, + "score": -0.23856120001818734 + } + }, + "time": "0:02:42.897265", + "diversity": { + "train": { + "mean_centered_corr": -0.017316685989499092, + "mean_pairwise_disagreement": NaN, + "member_std": 0.050752341747283936 + }, + "val": { + "mean_centered_corr": -0.014526012353599072, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017978072166443 + }, + "test": { + "mean_centered_corr": -0.008432705886662006, + "mean_pairwise_disagreement": NaN, + "member_std": 0.13299168646335602 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27, + 4, + 21, + 22, + 7, + 31, + 19, + 23, + 24, + 26 + ], + "head_selection_time": "0:00:00.159301" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..61abe5749 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,236 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22732156992730132, + "mae": 0.12432415038347244, + "r2": 0.6698622703552246, + "score": -0.22732156992730132 + }, + "val": { + "rmse": 0.23962393015909736, + "mae": 0.13407635688781738, + "r2": 0.6164591312408447, + "score": -0.23962393015909736 + }, + "test": { + "rmse": 0.24513925399849143, + "mae": 0.16054488718509674, + "r2": 0.5893561840057373, + "score": -0.24513925399849143 + } + }, + "time": "0:01:44.775403", + "diversity": { + "train": { + "mean_centered_corr": -0.01788729801774025, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04960332438349724 + }, + "val": { + "mean_centered_corr": -0.01788644678890705, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07234996557235718 + }, + "test": { + "mean_centered_corr": -0.018943432718515396, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09170374274253845 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 12 + ], + "head_selection_time": "0:00:00.015426" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0.toml new file mode 100644 index 000000000..3060f47c8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0.toml @@ -0,0 +1,57 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0/report.json new file mode 100644 index 000000000..202e55306 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/0/report.json @@ -0,0 +1,232 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.221803185432639, + "mae": 0.1178557351231575, + "r2": 0.6856963634490967, + "score": -0.221803185432639 + }, + "val": { + "rmse": 0.23493017564475285, + "mae": 0.12757235765457153, + "r2": 0.631337583065033, + "score": -0.23493017564475285 + }, + "test": { + "rmse": 0.23218164932047, + "mae": 0.14360341429710388, + "r2": 0.6316205263137817, + "score": -0.23218164932047 + } + }, + "time": "0:01:44.775403", + "diversity": { + "train": { + "mean_centered_corr": -0.01788729801774025, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04960332438349724 + }, + "val": { + "mean_centered_corr": -0.01788644678890705, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07234996557235718 + }, + "test": { + "mean_centered_corr": -0.018943432718515396, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09170374274253845 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d5ae81ee8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,246 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_mfb-evaluation/_esam_diagnostics/esam_mfb-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22279034307776457, + "mae": 0.11827730387449265, + "r2": 0.6828924417495728, + "score": -0.22279034307776457 + }, + "val": { + "rmse": 0.2336977870530739, + "mae": 0.12728972733020782, + "r2": 0.635195255279541, + "score": -0.2336977870530739 + }, + "test": { + "rmse": 0.23066386352840565, + "mae": 0.13997237384319305, + "r2": 0.6364210844039917, + "score": -0.23066386352840565 + } + }, + "time": "0:01:44.775403", + "diversity": { + "train": { + "mean_centered_corr": -0.01788729801774025, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04960332438349724 + }, + "val": { + "mean_centered_corr": -0.01788644678890705, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07234996557235718 + }, + "test": { + "mean_centered_corr": -0.018943432718515396, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09170374274253845 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 12, + 31, + 24, + 20, + 8, + 26, + 19, + 21, + 25, + 24, + 31 + ], + "head_selection_time": "0:00:00.175209" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..fc71d7ecb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-best-head-evaluation/0/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2255253318194067, + "mae": 0.1253618746995926, + "r2": 0.6750589609146118, + "score": -0.2255253318194067 + }, + "val": { + "rmse": 0.23949031043262045, + "mae": 0.13341857492923737, + "r2": 0.6168867349624634, + "score": -0.23949031043262045 + }, + "test": { + "rmse": 0.24155888192007524, + "mae": 0.14752615988254547, + "r2": 0.6012638807296753, + "score": -0.24155888192007524 + } + }, + "time": "0:01:30.183340", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015040" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0.toml new file mode 100644 index 000000000..758520650 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0/report.json new file mode 100644 index 000000000..bdc1bdf6a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/0/report.json @@ -0,0 +1,196 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2197185345348766, + "mae": 0.11797432601451874, + "r2": 0.6915766000747681, + "score": -0.2197185345348766 + }, + "val": { + "rmse": 0.23467774706155475, + "mae": 0.1267256885766983, + "r2": 0.632129430770874, + "score": -0.23467774706155475 + }, + "test": { + "rmse": 0.23716614057842658, + "mae": 0.1508055180311203, + "r2": 0.6156339645385742, + "score": -0.23716614057842658 + } + }, + "time": "0:01:30.183340", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0855a0948 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,212 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.001-evaluation/_esam_diagnostics/esam_rho0.001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22057000931073897, + "mae": 0.11852128803730011, + "r2": 0.6891815662384033, + "score": -0.22057000931073897 + }, + "val": { + "rmse": 0.23316146280351166, + "mae": 0.12596334517002106, + "r2": 0.6368677616119385, + "score": -0.23316146280351166 + }, + "test": { + "rmse": 0.23452524296935331, + "mae": 0.14668044447898865, + "r2": 0.6241463422775269, + "score": -0.23452524296935331 + } + }, + "time": "0:01:30.183340", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 4, + 16, + 12, + 0, + 25, + 31, + 3, + 22, + 29, + 12, + 27, + 20 + ], + "head_selection_time": "0:00:00.202696" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bc82de281 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-best-head-evaluation/0/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2261742520907439, + "mae": 0.12224072217941284, + "r2": 0.6731863021850586, + "score": -0.2261742520907439 + }, + "val": { + "rmse": 0.2377136850270048, + "mae": 0.13327811658382416, + "r2": 0.622549831867218, + "score": -0.2377136850270048 + }, + "test": { + "rmse": 0.262345673404661, + "mae": 0.17990553379058838, + "r2": 0.5296865105628967, + "score": -0.262345673404661 + } + }, + "time": "0:01:27.301753", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015122" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0.toml new file mode 100644 index 000000000..05c49e679 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0/report.json new file mode 100644 index 000000000..5fee35a30 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0/report.json @@ -0,0 +1,196 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22265827445732783, + "mae": 0.11788544058799744, + "r2": 0.6832683086395264, + "score": -0.22265827445732783 + }, + "val": { + "rmse": 0.23474623366768377, + "mae": 0.1270996779203415, + "r2": 0.6319146752357483, + "score": -0.23474623366768377 + }, + "test": { + "rmse": 0.23936563514587833, + "mae": 0.15535929799079895, + "r2": 0.6084716320037842, + "score": -0.23936563514587833 + } + }, + "time": "0:01:27.301753", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9ca33d310 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/_esam_diagnostics/esam_rho0.0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22450348389711117, + "mae": 0.11922673135995865, + "r2": 0.6779968738555908, + "score": -0.22450348389711117 + }, + "val": { + "rmse": 0.23295802298295287, + "mae": 0.1258699893951416, + "r2": 0.6375011801719666, + "score": -0.23295802298295287 + }, + "test": { + "rmse": 0.23903366020873654, + "mae": 0.15511320531368256, + "r2": 0.6095569133758545, + "score": -0.23903366020873654 + } + }, + "time": "0:01:27.301753", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 22, + 12, + 16, + 21, + 19, + 25, + 29 + ], + "head_selection_time": "0:00:00.128997" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5ac0f4a22 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-best-head-evaluation/0/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2276013825171354, + "mae": 0.12862664461135864, + "r2": 0.6690490245819092, + "score": -0.2276013825171354 + }, + "val": { + "rmse": 0.2398028724080995, + "mae": 0.1366567611694336, + "r2": 0.6158860921859741, + "score": -0.2398028724080995 + }, + "test": { + "rmse": 0.23711371881317633, + "mae": 0.14537189900875092, + "r2": 0.6158038973808289, + "score": -0.23711371881317633 + } + }, + "time": "0:01:29.953970", + "diversity": { + "train": { + "mean_centered_corr": -0.017507946118712425, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0488579161465168 + }, + "val": { + "mean_centered_corr": -0.017691537737846375, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0768243595957756 + }, + "test": { + "mean_centered_corr": -0.014683594927191734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10134773701429367 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015185" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0.toml new file mode 100644 index 000000000..f5a2c1fd1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0/report.json new file mode 100644 index 000000000..393470cd9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/0/report.json @@ -0,0 +1,196 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21979515700408836, + "mae": 0.12170511484146118, + "r2": 0.6913614869117737, + "score": -0.21979515700408836 + }, + "val": { + "rmse": 0.23533073111501152, + "mae": 0.12932802736759186, + "r2": 0.6300793886184692, + "score": -0.23533073111501152 + }, + "test": { + "rmse": 0.24162357572341994, + "mae": 0.15881586074829102, + "r2": 0.6010502576828003, + "score": -0.24162357572341994 + } + }, + "time": "0:01:29.953970", + "diversity": { + "train": { + "mean_centered_corr": -0.017507946118712425, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0488579161465168 + }, + "val": { + "mean_centered_corr": -0.017691537737846375, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0768243595957756 + }, + "test": { + "mean_centered_corr": -0.014683594927191734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10134773701429367 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..6fcc28b95 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.005-evaluation/_esam_diagnostics/esam_rho0.005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22114994595231807, + "mae": 0.12296636402606964, + "r2": 0.6875449419021606, + "score": -0.22114994595231807 + }, + "val": { + "rmse": 0.23380275644783471, + "mae": 0.12705549597740173, + "r2": 0.6348674893379211, + "score": -0.23380275644783471 + }, + "test": { + "rmse": 0.24257603366460162, + "mae": 0.1593162715435028, + "r2": 0.5978988409042358, + "score": -0.24257603366460162 + } + }, + "time": "0:01:29.953970", + "diversity": { + "train": { + "mean_centered_corr": -0.017507946118712425, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0488579161465168 + }, + "val": { + "mean_centered_corr": -0.017691537737846375, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0768243595957756 + }, + "test": { + "mean_centered_corr": -0.014683594927191734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10134773701429367 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 25, + 0, + 16, + 30, + 22, + 25, + 31, + 16, + 22 + ], + "head_selection_time": "0:00:00.158575" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c9c43f347 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-best-head-evaluation/0/report.json @@ -0,0 +1,200 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2265011030074498, + "mae": 0.12218555063009262, + "r2": 0.6722410917282104, + "score": -0.2265011030074498 + }, + "val": { + "rmse": 0.23855772550880586, + "mae": 0.1326008141040802, + "r2": 0.6198647022247314, + "score": -0.23855772550880586 + }, + "test": { + "rmse": 0.24435949051951408, + "mae": 0.1609477698802948, + "r2": 0.5919644832611084, + "score": -0.24435949051951408 + } + }, + "time": "0:01:26.741342", + "diversity": { + "train": { + "mean_centered_corr": -0.018773315474390984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047917984426021576 + }, + "val": { + "mean_centered_corr": -0.01945357583463192, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06856771558523178 + }, + "test": { + "mean_centered_corr": -0.015788640826940536, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09582500904798508 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 18 + ], + "head_selection_time": "0:00:00.015262" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0.toml new file mode 100644 index 000000000..1f58fca60 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0.toml @@ -0,0 +1,44 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.01 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0/report.json new file mode 100644 index 000000000..95ee3f701 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/0/report.json @@ -0,0 +1,196 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22394801918202625, + "mae": 0.12045823782682419, + "r2": 0.6795883178710938, + "score": -0.22394801918202625 + }, + "val": { + "rmse": 0.2345959019738195, + "mae": 0.12734748423099518, + "r2": 0.6323859691619873, + "score": -0.2345959019738195 + }, + "test": { + "rmse": 0.23852799100834732, + "mae": 0.15464624762535095, + "r2": 0.611207127571106, + "score": -0.23852799100834732 + } + }, + "time": "0:01:26.741342", + "diversity": { + "train": { + "mean_centered_corr": -0.018773315474390984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047917984426021576 + }, + "val": { + "mean_centered_corr": -0.01945357583463192, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06856771558523178 + }, + "test": { + "mean_centered_corr": -0.015788640826940536, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09582500904798508 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..51c3bd29e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,206 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/esam_rho0.01-evaluation/_esam_diagnostics/esam_rho0.01-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2237227812786278, + "mae": 0.12099222093820572, + "r2": 0.6802325248718262, + "score": -0.2237227812786278 + }, + "val": { + "rmse": 0.23321113093450146, + "mae": 0.12660610675811768, + "r2": 0.6367130279541016, + "score": -0.23321113093450146 + }, + "test": { + "rmse": 0.23887208479744804, + "mae": 0.15645892918109894, + "r2": 0.610084593296051, + "score": -0.23887208479744804 + } + }, + "time": "0:01:26.741342", + "diversity": { + "train": { + "mean_centered_corr": -0.018773315474390984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047917984426021576 + }, + "val": { + "mean_centered_corr": -0.01945357583463192, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06856771558523178 + }, + "test": { + "mean_centered_corr": -0.015788640826940536, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09582500904798508 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 18, + 4, + 19, + 1, + 20, + 21, + 25 + ], + "head_selection_time": "0:00:00.115142" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..27e477e2c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,679 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.23359725177653518, + "mae": 0.12286694347858429, + "r2": 0.651382327079773, + "score": -0.23359725177653518 + }, + "val": { + "rmse": 0.23765498863853576, + "mae": 0.13634204864501953, + "r2": 0.6227362155914307, + "score": -0.23765498863853576 + }, + "test": { + "rmse": 0.2536206620016002, + "mae": 0.175298810005188, + "r2": 0.5604493618011475, + "score": -0.2536206620016002 + } + }, + "time": "0:01:22.331704", + "diversity": { + "train": { + "mean_centered_corr": -0.018672151491045952, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0502934493124485 + }, + "val": { + "mean_centered_corr": -0.01943311281502247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06968922913074493 + }, + "test": { + "mean_centered_corr": -0.020102957263588905, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09299962222576141 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015321" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..e9f43e127 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0.toml @@ -0,0 +1,58 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..78c9ce2d6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0/report.json @@ -0,0 +1,675 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22208876822485193, + "mae": 0.11730500310659409, + "r2": 0.6848864555358887, + "score": -0.22208876822485193 + }, + "val": { + "rmse": 0.23508488120210985, + "mae": 0.12783657014369965, + "r2": 0.6308518648147583, + "score": -0.23508488120210985 + }, + "test": { + "rmse": 0.23568789402876444, + "mae": 0.14989204704761505, + "r2": 0.6204105615615845, + "score": -0.23568789402876444 + } + }, + "time": "0:01:22.331704", + "diversity": { + "train": { + "mean_centered_corr": -0.018672151491045952, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0502934493124485 + }, + "val": { + "mean_centered_corr": -0.01943311281502247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06968922913074493 + }, + "test": { + "mean_centered_corr": -0.020102957263588905, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09299962222576141 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..635150edd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22343022528056483, + "mae": 0.11760932952165604, + "r2": 0.6810683012008667, + "score": -0.22343022528056483 + }, + "val": { + "rmse": 0.23320900639668657, + "mae": 0.12497496604919434, + "r2": 0.6367197036743164, + "score": -0.23320900639668657 + }, + "test": { + "rmse": 0.23362227205150313, + "mae": 0.14383481442928314, + "r2": 0.6270350217819214, + "score": -0.23362227205150313 + } + }, + "time": "0:01:22.331704", + "diversity": { + "train": { + "mean_centered_corr": -0.018672151491045952, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0502934493124485 + }, + "val": { + "mean_centered_corr": -0.01943311281502247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06968922913074493 + }, + "test": { + "mean_centered_corr": -0.020102957263588905, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09299962222576141 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 20, + 29, + 19, + 0, + 12, + 4, + 20, + 21, + 19 + ], + "head_selection_time": "0:00:00.159678" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-best-head-evaluation/0/report.json new file mode 100644 index 000000000..adca48472 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-best-head-evaluation/0/report.json @@ -0,0 +1,226 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7039222121238708, + "min_features_per_member": 253.0, + "mean_features_per_member": 275.9375, + "max_features_per_member": 294.0, + "min_members_per_feature": 15.0, + "mean_members_per_feature": 22.525510787963867, + "max_members_per_feature": 30.0, + "mean_pairwise_jaccard": 0.542541533357324, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.23190653592940091, + "mae": 0.12970374524593353, + "r2": 0.6564105153083801, + "score": -0.23190653592940091 + }, + "val": { + "rmse": 0.24062365494389312, + "mae": 0.13845081627368927, + "r2": 0.6132521629333496, + "score": -0.24062365494389312 + }, + "test": { + "rmse": 0.24557535127152327, + "mae": 0.16125304996967316, + "r2": 0.5878938436508179, + "score": -0.24557535127152327 + } + }, + "time": "0:00:59.832262", + "diversity": { + "train": { + "mean_centered_corr": -0.018218370154500008, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051916833966970444 + }, + "val": { + "mean_centered_corr": -0.019246703013777733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07262936979532242 + }, + "test": { + "mean_centered_corr": -0.017951499670743942, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09685395658016205 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015144" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0.toml new file mode 100644 index 000000000..7fa20abf9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.7 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0/report.json new file mode 100644 index 000000000..e3eaa700c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-evaluation/0/report.json @@ -0,0 +1,222 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7039222121238708, + "min_features_per_member": 253.0, + "mean_features_per_member": 275.9375, + "max_features_per_member": 294.0, + "min_members_per_feature": 15.0, + "mean_members_per_feature": 22.525510787963867, + "max_members_per_feature": 30.0, + "mean_pairwise_jaccard": 0.542541533357324, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22535369940802735, + "mae": 0.11960747092962265, + "r2": 0.6755533218383789, + "score": -0.22535369940802735 + }, + "val": { + "rmse": 0.23535581244592554, + "mae": 0.12930907309055328, + "r2": 0.6300005316734314, + "score": -0.23535581244592554 + }, + "test": { + "rmse": 0.23562693018337708, + "mae": 0.14956335723400116, + "r2": 0.6206068992614746, + "score": -0.23562693018337708 + } + }, + "time": "0:00:59.832262", + "diversity": { + "train": { + "mean_centered_corr": -0.018218370154500008, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051916833966970444 + }, + "val": { + "mean_centered_corr": -0.019246703013777733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07262936979532242 + }, + "test": { + "mean_centered_corr": -0.017951499670743942, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09685395658016205 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3bb540523 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.7-greedy-heads-evaluation/0/report.json @@ -0,0 +1,236 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.7, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.7039222121238708, + "min_features_per_member": 253.0, + "mean_features_per_member": 275.9375, + "max_features_per_member": 294.0, + "min_members_per_feature": 15.0, + "mean_members_per_feature": 22.525510787963867, + "max_members_per_feature": 30.0, + "mean_pairwise_jaccard": 0.542541533357324, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22381426184098463, + "mae": 0.11932115256786346, + "r2": 0.6799709796905518, + "score": -0.22381426184098463 + }, + "val": { + "rmse": 0.23323099362803706, + "mae": 0.12578125298023224, + "r2": 0.6366511583328247, + "score": -0.23323099362803706 + }, + "test": { + "rmse": 0.2357129689640159, + "mae": 0.149356409907341, + "r2": 0.620329737663269, + "score": -0.2357129689640159 + } + }, + "time": "0:00:59.832262", + "diversity": { + "train": { + "mean_centered_corr": -0.018218370154500008, + "mean_pairwise_disagreement": NaN, + "member_std": 0.051916833966970444 + }, + "val": { + "mean_centered_corr": -0.019246703013777733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07262936979532242 + }, + "test": { + "mean_centered_corr": -0.017951499670743942, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09685395658016205 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 2, + 28, + 20, + 1, + 19, + 21, + 14, + 31, + 20, + 12 + ], + "head_selection_time": "0:00:00.173195" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-best-head-evaluation/0/report.json new file mode 100644 index 000000000..688b587ad --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-best-head-evaluation/0/report.json @@ -0,0 +1,226 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8038105964660645, + "min_features_per_member": 298.0, + "mean_features_per_member": 315.09375, + "max_features_per_member": 335.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.721939086914062, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6715097166219348, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2316786318343145, + "mae": 0.12621846795082092, + "r2": 0.6570855379104614, + "score": -0.2316786318343145 + }, + "val": { + "rmse": 0.2380346366558266, + "mae": 0.1380215585231781, + "r2": 0.6215299367904663, + "score": -0.2380346366558266 + }, + "test": { + "rmse": 0.2579307429494123, + "mae": 0.17571206390857697, + "r2": 0.5453828573226929, + "score": -0.2579307429494123 + } + }, + "time": "0:00:59.854964", + "diversity": { + "train": { + "mean_centered_corr": -0.017583154141902924, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05003640055656433 + }, + "val": { + "mean_centered_corr": -0.018760371953248978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07571490854024887 + }, + "test": { + "mean_centered_corr": -0.019624220207333565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09894119203090668 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.016145" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0.toml new file mode 100644 index 000000000..d44bc73bf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.8 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0/report.json new file mode 100644 index 000000000..66077afb4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0/report.json @@ -0,0 +1,222 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8038105964660645, + "min_features_per_member": 298.0, + "mean_features_per_member": 315.09375, + "max_features_per_member": 335.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.721939086914062, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6715097166219348, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22081238228152064, + "mae": 0.11986075341701508, + "r2": 0.6884980797767639, + "score": -0.22081238228152064 + }, + "val": { + "rmse": 0.2341279953511937, + "mae": 0.12804140150547028, + "r2": 0.6338509321212769, + "score": -0.2341279953511937 + }, + "test": { + "rmse": 0.23402994505001048, + "mae": 0.14647243916988373, + "r2": 0.6257321834564209, + "score": -0.23402994505001048 + } + }, + "time": "0:00:59.854964", + "diversity": { + "train": { + "mean_centered_corr": -0.017583154141902924, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05003640055656433 + }, + "val": { + "mean_centered_corr": -0.018760371953248978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07571490854024887 + }, + "test": { + "mean_centered_corr": -0.019624220207333565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09894119203090668 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b613a292e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-greedy-heads-evaluation/0/report.json @@ -0,0 +1,235 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.8, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.8038105964660645, + "min_features_per_member": 298.0, + "mean_features_per_member": 315.09375, + "max_features_per_member": 335.0, + "min_members_per_feature": 19.0, + "mean_members_per_feature": 25.721939086914062, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.6715097166219348, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22132344126163775, + "mae": 0.11854095757007599, + "r2": 0.6870545148849487, + "score": -0.22132344126163775 + }, + "val": { + "rmse": 0.23237596587359466, + "mae": 0.1258111149072647, + "r2": 0.639310359954834, + "score": -0.23237596587359466 + }, + "test": { + "rmse": 0.23064696162155762, + "mae": 0.1411736011505127, + "r2": 0.6364743709564209, + "score": -0.23064696162155762 + } + }, + "time": "0:00:59.854964", + "diversity": { + "train": { + "mean_centered_corr": -0.017583154141902924, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05003640055656433 + }, + "val": { + "mean_centered_corr": -0.018760371953248978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07571490854024887 + }, + "test": { + "mean_centered_corr": -0.019624220207333565, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09894119203090668 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 28, + 2, + 27, + 12, + 24, + 9, + 3, + 21, + 28 + ], + "head_selection_time": "0:00:00.166841" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ed475ec53 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-best-head-evaluation/0/report.json @@ -0,0 +1,226 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.2105902710250961, + "mae": 0.11857529729604721, + "r2": 0.716671347618103, + "score": -0.2105902710250961 + }, + "val": { + "rmse": 0.24356476580119216, + "mae": 0.1343512386083603, + "r2": 0.6037400364875793, + "score": -0.24356476580119216 + }, + "test": { + "rmse": 0.2459548830871388, + "mae": 0.14966677129268646, + "r2": 0.5866190195083618, + "score": -0.2459548830871388 + } + }, + "time": "0:01:08.783577", + "diversity": { + "train": { + "mean_centered_corr": -0.016982607543468475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05261818319559097 + }, + "val": { + "mean_centered_corr": -0.017025893554091454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08680207282304764 + }, + "test": { + "mean_centered_corr": -0.01696028746664524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11552654951810837 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31 + ], + "head_selection_time": "0:00:00.015090" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0.toml new file mode 100644 index 000000000..da979e186 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0/report.json new file mode 100644 index 000000000..61c8effcd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-evaluation/0/report.json @@ -0,0 +1,222 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20459342170445438, + "mae": 0.11115512996912003, + "r2": 0.732577919960022, + "score": -0.20459342170445438 + }, + "val": { + "rmse": 0.23520949733542967, + "mae": 0.1269780397415161, + "r2": 0.6304603815078735, + "score": -0.23520949733542967 + }, + "test": { + "rmse": 0.23880983570262945, + "mae": 0.15338461101055145, + "r2": 0.6102877855300903, + "score": -0.23880983570262945 + } + }, + "time": "0:01:08.783577", + "diversity": { + "train": { + "mean_centered_corr": -0.016982607543468475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05261818319559097 + }, + "val": { + "mean_centered_corr": -0.017025893554091454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08680207282304764 + }, + "test": { + "mean_centered_corr": -0.01696028746664524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11552654951810837 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8321a1cc8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.9-greedy-heads-evaluation/0/report.json @@ -0,0 +1,234 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20332438513408257, + "mae": 0.11058838665485382, + "r2": 0.7358851432800293, + "score": -0.20332438513408257 + }, + "val": { + "rmse": 0.233840053737128, + "mae": 0.12618696689605713, + "r2": 0.6347509622573853, + "score": -0.233840053737128 + }, + "test": { + "rmse": 0.23548819505307594, + "mae": 0.14876849949359894, + "r2": 0.6210535764694214, + "score": -0.23548819505307594 + } + }, + "time": "0:01:08.783577", + "diversity": { + "train": { + "mean_centered_corr": -0.016982607543468475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05261818319559097 + }, + "val": { + "mean_centered_corr": -0.017025893554091454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08680207282304764 + }, + "test": { + "mean_centered_corr": -0.01696028746664524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11552654951810837 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 31, + 27, + 25, + 19, + 8, + 4, + 28, + 17, + 15 + ], + "head_selection_time": "0:00:00.144349" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b8b7f9147 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-best-head-evaluation/0/report.json @@ -0,0 +1,226 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22956111765569162, + "mae": 0.12580275535583496, + "r2": 0.6633252501487732, + "score": -0.22956111765569162 + }, + "val": { + "rmse": 0.23835928999137548, + "mae": 0.13410992920398712, + "r2": 0.6204968094825745, + "score": -0.23835928999137548 + }, + "test": { + "rmse": 0.27696045729988356, + "mae": 0.19395296275615692, + "r2": 0.4758263826370239, + "score": -0.27696045729988356 + } + }, + "time": "0:01:00.082091", + "diversity": { + "train": { + "mean_centered_corr": -0.017180385068058968, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04938776046037674 + }, + "val": { + "mean_centered_corr": -0.018672365695238113, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06963518261909485 + }, + "test": { + "mean_centered_corr": -0.01587279699742794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09464479982852936 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015230" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0.toml new file mode 100644 index 000000000..035cc781b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0/report.json new file mode 100644 index 000000000..a9a095271 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-evaluation/0/report.json @@ -0,0 +1,222 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21949891059220633, + "mae": 0.11622610688209534, + "r2": 0.6921929121017456, + "score": -0.21949891059220633 + }, + "val": { + "rmse": 0.2352128391676174, + "mae": 0.12513305246829987, + "r2": 0.6304498910903931, + "score": -0.2352128391676174 + }, + "test": { + "rmse": 0.23206813724553327, + "mae": 0.14319945871829987, + "r2": 0.6319806575775146, + "score": -0.23206813724553327 + } + }, + "time": "0:01:00.082091", + "diversity": { + "train": { + "mean_centered_corr": -0.017180385068058968, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04938776046037674 + }, + "val": { + "mean_centered_corr": -0.018672365695238113, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06963518261909485 + }, + "test": { + "mean_centered_corr": -0.01587279699742794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09464479982852936 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f5a5ce75c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/mfb_keep0.95-greedy-heads-evaluation/0/report.json @@ -0,0 +1,235 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21898836534384564, + "mae": 0.1169113889336586, + "r2": 0.6936231255531311, + "score": -0.21898836534384564 + }, + "val": { + "rmse": 0.23254505067097037, + "mae": 0.12637269496917725, + "r2": 0.6387852430343628, + "score": -0.23254505067097037 + }, + "test": { + "rmse": 0.23506758402869707, + "mae": 0.1480458676815033, + "r2": 0.622406005859375, + "score": -0.23506758402869707 + } + }, + "time": "0:01:00.082091", + "diversity": { + "train": { + "mean_centered_corr": -0.017180385068058968, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04938776046037674 + }, + "val": { + "mean_centered_corr": -0.018672365695238113, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06963518261909485 + }, + "test": { + "mean_centered_corr": -0.01587279699742794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09464479982852936 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 4, + 12, + 19, + 0, + 21, + 4, + 20, + 31, + 0 + ], + "head_selection_time": "0:00:00.161207" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c1b4cfc5c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-best-head-evaluation/0/report.json @@ -0,0 +1,656 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21924836575226364, + "mae": 0.12353650480508804, + "r2": 0.6928951740264893, + "score": -0.21924836575226364 + }, + "val": { + "rmse": 0.23788056532412807, + "mae": 0.13566423952579498, + "r2": 0.6220196485519409, + "score": -0.23788056532412807 + }, + "test": { + "rmse": 0.24833117735063384, + "mae": 0.1667243391275406, + "r2": 0.5785926580429077, + "score": -0.24833117735063384 + } + }, + "time": "0:02:05.644832", + "diversity": { + "train": { + "mean_centered_corr": -0.017985021695494652, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047170910984277725 + }, + "val": { + "mean_centered_corr": -0.017544301226735115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06983517855405807 + }, + "test": { + "mean_centered_corr": -0.015103532001376152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10299047082662582 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015293" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0.toml new file mode 100644 index 000000000..d46520305 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.1 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0/report.json new file mode 100644 index 000000000..58438d978 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21744429514871078, + "mae": 0.11533833295106888, + "r2": 0.6979283690452576, + "score": -0.21744429514871078 + }, + "val": { + "rmse": 0.23507577716825093, + "mae": 0.12675879895687103, + "r2": 0.6308804750442505, + "score": -0.23507577716825093 + }, + "test": { + "rmse": 0.2401363436403948, + "mae": 0.15580184757709503, + "r2": 0.6059463024139404, + "score": -0.2401363436403948 + } + }, + "time": "0:02:05.644832", + "diversity": { + "train": { + "mean_centered_corr": -0.017985021695494652, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047170910984277725 + }, + "val": { + "mean_centered_corr": -0.017544301226735115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06983517855405807 + }, + "test": { + "mean_centered_corr": -0.015103532001376152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10299047082662582 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..be77993ab --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_cf_fisd-greedy-heads-evaluation/0/report.json @@ -0,0 +1,668 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.1, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21849668686011264, + "mae": 0.11518727242946625, + "r2": 0.6949973702430725, + "score": -0.21849668686011264 + }, + "val": { + "rmse": 0.233603375533187, + "mae": 0.12658587098121643, + "r2": 0.6354899406433105, + "score": -0.233603375533187 + }, + "test": { + "rmse": 0.23596160620261294, + "mae": 0.14986030757427216, + "r2": 0.6195283532142639, + "score": -0.23596160620261294 + } + }, + "time": "0:02:05.644832", + "diversity": { + "train": { + "mean_centered_corr": -0.017985021695494652, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047170910984277725 + }, + "val": { + "mean_centered_corr": -0.017544301226735115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06983517855405807 + }, + "test": { + "mean_centered_corr": -0.015103532001376152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10299047082662582 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 4, + 26, + 9, + 1, + 6, + 22, + 4, + 19, + 0, + 26, + 20, + 29 + ], + "head_selection_time": "0:00:00.204485" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8e988752a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2353728431166263, + "mae": 0.12022902816534042, + "r2": 0.6460624933242798, + "score": -0.2353728431166263 + }, + "val": { + "rmse": 0.2398484083586863, + "mae": 0.13140499591827393, + "r2": 0.615740180015564, + "score": -0.2398484083586863 + }, + "test": { + "rmse": 0.2374880282622196, + "mae": 0.15091650187969208, + "r2": 0.6145899295806885, + "score": -0.2374880282622196 + } + }, + "time": "0:02:41.758491", + "diversity": { + "train": { + "mean_centered_corr": -0.01889662630856037, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04478425905108452 + }, + "val": { + "mean_centered_corr": -0.02039206400513649, + "mean_pairwise_disagreement": NaN, + "member_std": 0.063406802713871 + }, + "test": { + "mean_centered_corr": -0.01742168515920639, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0834045484662056 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28 + ], + "head_selection_time": "0:00:00.015316" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0.toml new file mode 100644 index 000000000..d960be7a0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0/report.json new file mode 100644 index 000000000..4c0caaef2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23095795064666533, + "mae": 0.12093835324048996, + "r2": 0.6592155694961548, + "score": -0.23095795064666533 + }, + "val": { + "rmse": 0.23528677495791916, + "mae": 0.1253853291273117, + "r2": 0.6302175521850586, + "score": -0.23528677495791916 + }, + "test": { + "rmse": 0.23252163677391782, + "mae": 0.1432647854089737, + "r2": 0.6305409073829651, + "score": -0.23252163677391782 + } + }, + "time": "0:02:41.758491", + "diversity": { + "train": { + "mean_centered_corr": -0.01889662630856037, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04478425905108452 + }, + "val": { + "mean_centered_corr": -0.02039206400513649, + "mean_pairwise_disagreement": NaN, + "member_std": 0.063406802713871 + }, + "test": { + "mean_centered_corr": -0.01742168515920639, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0834045484662056 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7815e3439 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-greedy-heads-evaluation/0/report.json @@ -0,0 +1,217 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23231665048542627, + "mae": 0.12011752277612686, + "r2": 0.6551941633224487, + "score": -0.23231665048542627 + }, + "val": { + "rmse": 0.2336232925794531, + "mae": 0.12742893397808075, + "r2": 0.6354278326034546, + "score": -0.2336232925794531 + }, + "test": { + "rmse": 0.233028022164673, + "mae": 0.1476990431547165, + "r2": 0.6289299726486206, + "score": -0.233028022164673 + } + }, + "time": "0:02:41.758491", + "diversity": { + "train": { + "mean_centered_corr": -0.01889662630856037, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04478425905108452 + }, + "val": { + "mean_centered_corr": -0.02039206400513649, + "mean_pairwise_disagreement": NaN, + "member_std": 0.063406802713871 + }, + "test": { + "mean_centered_corr": -0.01742168515920639, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0834045484662056 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28, + 16, + 12, + 4, + 18 + ], + "head_selection_time": "0:00:00.087428" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f165e1fc0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-best-head-evaluation/0/report.json @@ -0,0 +1,239 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22945006775369764, + "mae": 0.12472175061702728, + "r2": 0.6636508703231812, + "score": -0.22945006775369764 + }, + "val": { + "rmse": 0.2399809438744672, + "mae": 0.13324783742427826, + "r2": 0.6153154373168945, + "score": -0.2399809438744672 + }, + "test": { + "rmse": 0.24194702392020134, + "mae": 0.15345057845115662, + "r2": 0.599981427192688, + "score": -0.24194702392020134 + } + }, + "time": "0:01:56.838272", + "diversity": { + "train": { + "mean_centered_corr": -0.018120070919394493, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04867309704422951 + }, + "val": { + "mean_centered_corr": -0.01935560256242752, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07195134460926056 + }, + "test": { + "mean_centered_corr": -0.019733797758817673, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09048754721879959 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015258" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0.toml new file mode 100644 index 000000000..25b3f5ef5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0.toml @@ -0,0 +1,54 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0/report.json new file mode 100644 index 000000000..59ddd06ea --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-evaluation/0/report.json @@ -0,0 +1,235 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2171101669154321, + "mae": 0.11589445918798447, + "r2": 0.698855996131897, + "score": -0.2171101669154321 + }, + "val": { + "rmse": 0.23551030167646542, + "mae": 0.12883572280406952, + "r2": 0.6295146346092224, + "score": -0.23551030167646542 + }, + "test": { + "rmse": 0.23422079576562096, + "mae": 0.14804303646087646, + "r2": 0.625121533870697, + "score": -0.23422079576562096 + } + }, + "time": "0:01:56.838272", + "diversity": { + "train": { + "mean_centered_corr": -0.018120070919394493, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04867309704422951 + }, + "val": { + "mean_centered_corr": -0.01935560256242752, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07195134460926056 + }, + "test": { + "mean_centered_corr": -0.019733797758817673, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09048754721879959 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..855c876e7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_mfb-greedy-heads-evaluation/0/report.json @@ -0,0 +1,245 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22016236214854085, + "mae": 0.11783046275377274, + "r2": 0.6903293132781982, + "score": -0.22016236214854085 + }, + "val": { + "rmse": 0.23358161473727912, + "mae": 0.1258193999528885, + "r2": 0.6355578899383545, + "score": -0.23358161473727912 + }, + "test": { + "rmse": 0.2330478525093287, + "mae": 0.14592212438583374, + "r2": 0.6288667917251587, + "score": -0.2330478525093287 + } + }, + "time": "0:01:56.838272", + "diversity": { + "train": { + "mean_centered_corr": -0.018120070919394493, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04867309704422951 + }, + "val": { + "mean_centered_corr": -0.01935560256242752, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07195134460926056 + }, + "test": { + "mean_centered_corr": -0.019733797758817673, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09048754721879959 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 10, + 12, + 19, + 4, + 24, + 8 + ], + "head_selection_time": "0:00:00.114671" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a2f79f3ed --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 962, + "metrics": { + "train": { + "rmse": 0.2188904430637261, + "mae": 0.1215578094124794, + "r2": 0.6938970685005188, + "score": -0.2188904430637261 + }, + "val": { + "rmse": 0.23946739670755524, + "mae": 0.13201551139354706, + "r2": 0.6169600486755371, + "score": -0.23946739670755524 + }, + "test": { + "rmse": 0.25239608328273316, + "mae": 0.16684766113758087, + "r2": 0.5646837949752808, + "score": -0.25239608328273316 + } + }, + "time": "0:01:13.625929", + "diversity": { + "train": { + "mean_centered_corr": -0.0177121814340353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04854941368103027 + }, + "val": { + "mean_centered_corr": -0.017364732921123505, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07598073035478592 + }, + "test": { + "mean_centered_corr": -0.012729885056614876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1117154061794281 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.014985" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..07fa7cfe7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..3bfe11f91 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 962, + "metrics": { + "train": { + "rmse": 0.2138111441702229, + "mae": 0.11529327929019928, + "r2": 0.7079383134841919, + "score": -0.2138111441702229 + }, + "val": { + "rmse": 0.23622468781228076, + "mae": 0.1262529343366623, + "r2": 0.6272635459899902, + "score": -0.23622468781228076 + }, + "test": { + "rmse": 0.23935440604862293, + "mae": 0.15282411873340607, + "r2": 0.6085084080696106, + "score": -0.23935440604862293 + } + }, + "time": "0:01:13.625929", + "diversity": { + "train": { + "mean_centered_corr": -0.0177121814340353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04854941368103027 + }, + "val": { + "mean_centered_corr": -0.017364732921123505, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07598073035478592 + }, + "test": { + "mean_centered_corr": -0.012729885056614876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1117154061794281 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..124eabc46 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,210 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 962, + "metrics": { + "train": { + "rmse": 0.21338362874058017, + "mae": 0.11555410176515579, + "r2": 0.7091051340103149, + "score": -0.21338362874058017 + }, + "val": { + "rmse": 0.23332344829675253, + "mae": 0.1253633350133896, + "r2": 0.6363630294799805, + "score": -0.23332344829675253 + }, + "test": { + "rmse": 0.24781019077451777, + "mae": 0.16565711796283722, + "r2": 0.5803589820861816, + "score": -0.24781019077451777 + } + }, + "time": "0:01:13.625929", + "diversity": { + "train": { + "mean_centered_corr": -0.0177121814340353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04854941368103027 + }, + "val": { + "mean_centered_corr": -0.017364732921123505, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07598073035478592 + }, + "test": { + "mean_centered_corr": -0.012729885056614876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1117154061794281 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 21, + 19, + 18, + 12, + 10, + 19, + 25 + ], + "head_selection_time": "0:00:00.129066" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..45688715d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22423689267064406, + "mae": 0.12748517096042633, + "r2": 0.6787611842155457, + "score": -0.22423689267064406 + }, + "val": { + "rmse": 0.2370483756746748, + "mae": 0.13395926356315613, + "r2": 0.6246596574783325, + "score": -0.2370483756746748 + }, + "test": { + "rmse": 0.23920040609688478, + "mae": 0.14310523867607117, + "r2": 0.6090120077133179, + "score": -0.23920040609688478 + } + }, + "time": "0:01:09.057982", + "diversity": { + "train": { + "mean_centered_corr": -0.019481390714645386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05119703337550163 + }, + "val": { + "mean_centered_corr": -0.01973012462258339, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07138136774301529 + }, + "test": { + "mean_centered_corr": -0.016405168920755386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09870413690805435 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015930" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..44f3d87b5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..475cad836 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22159317613989218, + "mae": 0.12286020070314407, + "r2": 0.6862912178039551, + "score": -0.22159317613989218 + }, + "val": { + "rmse": 0.2354682854519358, + "mae": 0.13157758116722107, + "r2": 0.6296467781066895, + "score": -0.2354682854519358 + }, + "test": { + "rmse": 0.24014251007009368, + "mae": 0.15741941332817078, + "r2": 0.6059260964393616, + "score": -0.24014251007009368 + } + }, + "time": "0:01:09.057982", + "diversity": { + "train": { + "mean_centered_corr": -0.019481390714645386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05119703337550163 + }, + "val": { + "mean_centered_corr": -0.01973012462258339, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07138136774301529 + }, + "test": { + "mean_centered_corr": -0.016405168920755386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09870413690805435 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..42d104b71 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,206 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22248795490129444, + "mae": 0.1261499673128128, + "r2": 0.6837526559829712, + "score": -0.22248795490129444 + }, + "val": { + "rmse": 0.2328613038082034, + "mae": 0.12715381383895874, + "r2": 0.6378021240234375, + "score": -0.2328613038082034 + }, + "test": { + "rmse": 0.23498425768312764, + "mae": 0.14776451885700226, + "r2": 0.6226736307144165, + "score": -0.23498425768312764 + } + }, + "time": "0:01:09.057982", + "diversity": { + "train": { + "mean_centered_corr": -0.019481390714645386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05119703337550163 + }, + "val": { + "mean_centered_corr": -0.01973012462258339, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07138136774301529 + }, + "test": { + "mean_centered_corr": -0.016405168920755386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09870413690805435 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 21, + 4, + 24 + ], + "head_selection_time": "0:00:00.076523" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0d8b2f6a0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22493170953564973, + "mae": 0.12546482682228088, + "r2": 0.6767673492431641, + "score": -0.22493170953564973 + }, + "val": { + "rmse": 0.2389693720816827, + "mae": 0.13471423089504242, + "r2": 0.6185516119003296, + "score": -0.2389693720816827 + }, + "test": { + "rmse": 0.24127702155259925, + "mae": 0.14682982861995697, + "r2": 0.6021938323974609, + "score": -0.24127702155259925 + } + }, + "time": "0:01:08.653647", + "diversity": { + "train": { + "mean_centered_corr": -0.018831178545951843, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048278551548719406 + }, + "val": { + "mean_centered_corr": -0.019186949357390404, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06771456450223923 + }, + "test": { + "mean_centered_corr": -0.018310436978936195, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09045998007059097 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015389" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..be03dc864 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..e835cfb48 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2214359001653139, + "mae": 0.1211930438876152, + "r2": 0.6867363452911377, + "score": -0.2214359001653139 + }, + "val": { + "rmse": 0.23574409357673579, + "mae": 0.13180093467235565, + "r2": 0.6287786960601807, + "score": -0.23574409357673579 + }, + "test": { + "rmse": 0.2418545849757855, + "mae": 0.16086849570274353, + "r2": 0.6002870202064514, + "score": -0.2418545849757855 + } + }, + "time": "0:01:08.653647", + "diversity": { + "train": { + "mean_centered_corr": -0.018831178545951843, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048278551548719406 + }, + "val": { + "mean_centered_corr": -0.019186949357390404, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06771456450223923 + }, + "test": { + "mean_centered_corr": -0.018310436978936195, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09045998007059097 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..a475471c7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank2_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2219257249740597, + "mae": 0.11780673265457153, + "r2": 0.6853489875793457, + "score": -0.2219257249740597 + }, + "val": { + "rmse": 0.23334154528971895, + "mae": 0.1271604299545288, + "r2": 0.636306643486023, + "score": -0.23334154528971895 + }, + "test": { + "rmse": 0.23451478286562477, + "mae": 0.1480785757303238, + "r2": 0.6241798400878906, + "score": -0.23451478286562477 + } + }, + "time": "0:01:08.653647", + "diversity": { + "train": { + "mean_centered_corr": -0.018831178545951843, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048278551548719406 + }, + "val": { + "mean_centered_corr": -0.019186949357390404, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06771456450223923 + }, + "test": { + "mean_centered_corr": -0.018310436978936195, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09045998007059097 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 21, + 12, + 18, + 8 + ], + "head_selection_time": "0:00:00.088954" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8bf3f316d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2357054301684337, + "mae": 0.12653841078281403, + "r2": 0.6450614929199219, + "score": -0.2357054301684337 + }, + "val": { + "rmse": 0.23765212789668771, + "mae": 0.13472485542297363, + "r2": 0.6227452754974365, + "score": -0.23765212789668771 + }, + "test": { + "rmse": 0.24778267915496135, + "mae": 0.16990186274051666, + "r2": 0.5804521441459656, + "score": -0.24778267915496135 + } + }, + "time": "0:01:34.341238", + "diversity": { + "train": { + "mean_centered_corr": -0.018528519198298454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045790109783411026 + }, + "val": { + "mean_centered_corr": -0.01869109645485878, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057747576385736465 + }, + "test": { + "mean_centered_corr": -0.017511537298560143, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07643643766641617 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 18 + ], + "head_selection_time": "0:00:00.015225" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..485d94624 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..6f40fec13 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2313646747453307, + "mae": 0.12118230015039444, + "r2": 0.6580142974853516, + "score": -0.2313646747453307 + }, + "val": { + "rmse": 0.2354418316334552, + "mae": 0.1258188933134079, + "r2": 0.6297299861907959, + "score": -0.2354418316334552 + }, + "test": { + "rmse": 0.23162903716515967, + "mae": 0.14173103868961334, + "r2": 0.6333720684051514, + "score": -0.23162903716515967 + } + }, + "time": "0:01:34.341238", + "diversity": { + "train": { + "mean_centered_corr": -0.018528519198298454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045790109783411026 + }, + "val": { + "mean_centered_corr": -0.01869109645485878, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057747576385736465 + }, + "test": { + "mean_centered_corr": -0.017511537298560143, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07643643766641617 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..722e24fe8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23082056510008958, + "mae": 0.12217259407043457, + "r2": 0.6596208810806274, + "score": -0.23082056510008958 + }, + "val": { + "rmse": 0.2336025622310011, + "mae": 0.12522470951080322, + "r2": 0.6354925036430359, + "score": -0.2336025622310011 + }, + "test": { + "rmse": 0.23389241285301454, + "mae": 0.14725922048091888, + "r2": 0.6261719465255737, + "score": -0.23389241285301454 + } + }, + "time": "0:01:34.341238", + "diversity": { + "train": { + "mean_centered_corr": -0.018528519198298454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045790109783411026 + }, + "val": { + "mean_centered_corr": -0.01869109645485878, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057747576385736465 + }, + "test": { + "mean_centered_corr": -0.017511537298560143, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07643643766641617 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 18, + 21, + 19, + 25, + 6, + 4 + ], + "head_selection_time": "0:00:00.101019" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bcb05991c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22153469839383894, + "mae": 0.12104755640029907, + "r2": 0.6864567995071411, + "score": -0.22153469839383894 + }, + "val": { + "rmse": 0.23958569850307038, + "mae": 0.13397271931171417, + "r2": 0.6165815591812134, + "score": -0.23958569850307038 + }, + "test": { + "rmse": 0.2417253037065434, + "mae": 0.1469961255788803, + "r2": 0.6007142066955566, + "score": -0.2417253037065434 + } + }, + "time": "0:01:45.212537", + "diversity": { + "train": { + "mean_centered_corr": -0.018073728308081627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047459591180086136 + }, + "val": { + "mean_centered_corr": -0.01855890266597271, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06974152475595474 + }, + "test": { + "mean_centered_corr": -0.018261540681123734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09257550537586212 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015445" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..8b8f1a027 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..f138b2a28 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2156155421767402, + "mae": 0.11583763360977173, + "r2": 0.7029880285263062, + "score": -0.2156155421767402 + }, + "val": { + "rmse": 0.23544592963840097, + "mae": 0.1267930269241333, + "r2": 0.6297171115875244, + "score": -0.23544592963840097 + }, + "test": { + "rmse": 0.23541433043431914, + "mae": 0.14889299869537354, + "r2": 0.6212911605834961, + "score": -0.23541433043431914 + } + }, + "time": "0:01:45.212537", + "diversity": { + "train": { + "mean_centered_corr": -0.018073728308081627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047459591180086136 + }, + "val": { + "mean_centered_corr": -0.01855890266597271, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06974152475595474 + }, + "test": { + "mean_centered_corr": -0.018261540681123734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09257550537586212 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..fb9bb20f1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21704166785572343, + "mae": 0.11682380735874176, + "r2": 0.6990460157394409, + "score": -0.21704166785572343 + }, + "val": { + "rmse": 0.23389653800469046, + "mae": 0.12599465250968933, + "r2": 0.6345745325088501, + "score": -0.23389653800469046 + }, + "test": { + "rmse": 0.23994098363644276, + "mae": 0.1547209620475769, + "r2": 0.6065871715545654, + "score": -0.23994098363644276 + } + }, + "time": "0:01:45.212537", + "diversity": { + "train": { + "mean_centered_corr": -0.018073728308081627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047459591180086136 + }, + "val": { + "mean_centered_corr": -0.01855890266597271, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06974152475595474 + }, + "test": { + "mean_centered_corr": -0.018261540681123734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09257550537586212 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 21, + 25, + 4, + 2 + ], + "head_selection_time": "0:00:00.088219" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..2502a80ef --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23729621024354838, + "mae": 0.13337191939353943, + "r2": 0.6402543783187866, + "score": -0.23729621024354838 + }, + "val": { + "rmse": 0.23804322063719607, + "mae": 0.1294262707233429, + "r2": 0.6215026378631592, + "score": -0.23804322063719607 + }, + "test": { + "rmse": 0.24414388272972984, + "mae": 0.15476657450199127, + "r2": 0.5926841497421265, + "score": -0.24414388272972984 + } + }, + "time": "0:01:34.559559", + "diversity": { + "train": { + "mean_centered_corr": -0.01910587027668953, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045081716030836105 + }, + "val": { + "mean_centered_corr": -0.01968017779290676, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0602603442966938 + }, + "test": { + "mean_centered_corr": -0.01622942090034485, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08117352426052094 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015307" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..6546774a1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..3f7ae16d5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23182174394925567, + "mae": 0.1216563880443573, + "r2": 0.6566617488861084, + "score": -0.23182174394925567 + }, + "val": { + "rmse": 0.23604436658345962, + "mae": 0.12681564688682556, + "r2": 0.6278324127197266, + "score": -0.23604436658345962 + }, + "test": { + "rmse": 0.2328903381745147, + "mae": 0.14401739835739136, + "r2": 0.6293683052062988, + "score": -0.2328903381745147 + } + }, + "time": "0:01:34.559559", + "diversity": { + "train": { + "mean_centered_corr": -0.01910587027668953, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045081716030836105 + }, + "val": { + "mean_centered_corr": -0.01968017779290676, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0602603442966938 + }, + "test": { + "mean_centered_corr": -0.01622942090034485, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08117352426052094 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..edde87039 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank4_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,211 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23216803497367422, + "mae": 0.12254077196121216, + "r2": 0.6556352376937866, + "score": -0.23216803497367422 + }, + "val": { + "rmse": 0.2335869733918295, + "mae": 0.12630410492420197, + "r2": 0.6355411410331726, + "score": -0.2335869733918295 + }, + "test": { + "rmse": 0.2353936312104005, + "mae": 0.15104448795318604, + "r2": 0.621357798576355, + "score": -0.2353936312104005 + } + }, + "time": "0:01:34.559559", + "diversity": { + "train": { + "mean_centered_corr": -0.01910587027668953, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045081716030836105 + }, + "val": { + "mean_centered_corr": -0.01968017779290676, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0602603442966938 + }, + "test": { + "mean_centered_corr": -0.01622942090034485, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08117352426052094 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 4, + 18, + 16, + 12, + 25, + 21, + 18, + 19 + ], + "head_selection_time": "0:00:00.145132" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..474cdf3b1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-best-head-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 6702952, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23628005807891533, + "mae": 0.12606193125247955, + "r2": 0.6433287858963013, + "score": -0.23628005807891533 + }, + "val": { + "rmse": 0.23612384794902308, + "mae": 0.13170304894447327, + "r2": 0.6275817155838013, + "score": -0.23612384794902308 + }, + "test": { + "rmse": 0.25131993238610145, + "mae": 0.1727743297815323, + "r2": 0.5683881044387817, + "score": -0.25131993238610145 + } + }, + "time": "0:02:37.152304", + "diversity": { + "train": { + "mean_centered_corr": -0.018440110608935356, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04351954534649849 + }, + "val": { + "mean_centered_corr": -0.019317032769322395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059053633362054825 + }, + "test": { + "mean_centered_corr": -0.019231578335165977, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07535656541585922 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 18 + ], + "head_selection_time": "0:00:00.015278" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0.toml new file mode 100644 index 000000000..766daf18b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0/report.json new file mode 100644 index 000000000..73100e216 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 6702952, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2319284940460109, + "mae": 0.12257123738527298, + "r2": 0.6563454270362854, + "score": -0.2319284940460109 + }, + "val": { + "rmse": 0.23560034396487695, + "mae": 0.12617181241512299, + "r2": 0.6292312741279602, + "score": -0.23560034396487695 + }, + "test": { + "rmse": 0.2314192197875428, + "mae": 0.1411750614643097, + "r2": 0.6340359449386597, + "score": -0.2314192197875428 + } + }, + "time": "0:02:37.152304", + "diversity": { + "train": { + "mean_centered_corr": -0.018440110608935356, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04351954534649849 + }, + "val": { + "mean_centered_corr": -0.019317032769322395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059053633362054825 + }, + "test": { + "mean_centered_corr": -0.019231578335165977, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07535656541585922 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0dc9b308b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.0001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,211 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 6702952, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23160647965627332, + "mae": 0.12231546640396118, + "r2": 0.6572990417480469, + "score": -0.23160647965627332 + }, + "val": { + "rmse": 0.23331262295778735, + "mae": 0.12554694712162018, + "r2": 0.6363967657089233, + "score": -0.23331262295778735 + }, + "test": { + "rmse": 0.2320760028721284, + "mae": 0.14371301233768463, + "r2": 0.6319557428359985, + "score": -0.2320760028721284 + } + }, + "time": "0:02:37.152304", + "diversity": { + "train": { + "mean_centered_corr": -0.018440110608935356, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04351954534649849 + }, + "val": { + "mean_centered_corr": -0.019317032769322395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059053633362054825 + }, + "test": { + "mean_centered_corr": -0.019231578335165977, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07535656541585922 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 18, + 25, + 19, + 28, + 31, + 21, + 6, + 25, + 4 + ], + "head_selection_time": "0:00:00.145454" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e2e44ae71 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-best-head-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 6702952, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2211915746692412, + "mae": 0.12369639426469803, + "r2": 0.687427282333374, + "score": -0.2211915746692412 + }, + "val": { + "rmse": 0.239120553017687, + "mae": 0.13442911207675934, + "r2": 0.6180688738822937, + "score": -0.239120553017687 + }, + "test": { + "rmse": 0.23826554512424344, + "mae": 0.14826232194900513, + "r2": 0.6120622158050537, + "score": -0.23826554512424344 + } + }, + "time": "0:02:54.576470", + "diversity": { + "train": { + "mean_centered_corr": -0.018232719972729683, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045184310525655746 + }, + "val": { + "mean_centered_corr": -0.019308097660541534, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06649133563041687 + }, + "test": { + "mean_centered_corr": -0.017781618982553482, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09422237426042557 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015104" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0.toml new file mode 100644 index 000000000..cfb125e1a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0/report.json new file mode 100644 index 000000000..46253077e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 6702952, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21571915616347168, + "mae": 0.11579738557338715, + "r2": 0.7027024626731873, + "score": -0.21571915616347168 + }, + "val": { + "rmse": 0.23625085686725053, + "mae": 0.12631435692310333, + "r2": 0.6271809935569763, + "score": -0.23625085686725053 + }, + "test": { + "rmse": 0.2355661245169954, + "mae": 0.147227481007576, + "r2": 0.620802640914917, + "score": -0.2355661245169954 + } + }, + "time": "0:02:54.576470", + "diversity": { + "train": { + "mean_centered_corr": -0.018232719972729683, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045184310525655746 + }, + "val": { + "mean_centered_corr": -0.019308097660541534, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06649133563041687 + }, + "test": { + "mean_centered_corr": -0.017781618982553482, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09422237426042557 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..faaf6ce89 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise0.001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,212 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 6702952, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21646584334087177, + "mae": 0.11596201360225677, + "r2": 0.7006407976150513, + "score": -0.21646584334087177 + }, + "val": { + "rmse": 0.23353637226653048, + "mae": 0.1261347234249115, + "r2": 0.6356990337371826, + "score": -0.23353637226653048 + }, + "test": { + "rmse": 0.23842460978700958, + "mae": 0.1537153124809265, + "r2": 0.6115440726280212, + "score": -0.23842460978700958 + } + }, + "time": "0:02:54.576470", + "diversity": { + "train": { + "mean_centered_corr": -0.018232719972729683, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045184310525655746 + }, + "val": { + "mean_centered_corr": -0.019308097660541534, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06649133563041687 + }, + "test": { + "mean_centered_corr": -0.017781618982553482, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09422237426042557 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 19, + 12, + 18, + 2, + 21, + 25, + 19, + 12, + 10 + ], + "head_selection_time": "0:00:00.158560" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8d7959fac --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-best-head-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 6702952, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22504498078493876, + "mae": 0.12070199847221375, + "r2": 0.6764416694641113, + "score": -0.22504498078493876 + }, + "val": { + "rmse": 0.23899585630204528, + "mae": 0.13757818937301636, + "r2": 0.6184670925140381, + "score": -0.23899585630204528 + }, + "test": { + "rmse": 0.25744912519923097, + "mae": 0.18095265328884125, + "r2": 0.5470789670944214, + "score": -0.25744912519923097 + } + }, + "time": "0:02:49.448996", + "diversity": { + "train": { + "mean_centered_corr": -0.018503453582525253, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04646626114845276 + }, + "val": { + "mean_centered_corr": -0.018042705953121185, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06875067204236984 + }, + "test": { + "mean_centered_corr": -0.016347019001841545, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09534233063459396 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 18 + ], + "head_selection_time": "0:00:00.015265" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0.toml new file mode 100644 index 000000000..82c69bc53 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0.toml @@ -0,0 +1,41 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 8 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0/report.json new file mode 100644 index 000000000..24a7a971e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-evaluation/0/report.json @@ -0,0 +1,199 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 6702952, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.21933724563493878, + "mae": 0.11967557668685913, + "r2": 0.6926461458206177, + "score": -0.21933724563493878 + }, + "val": { + "rmse": 0.23553038961691936, + "mae": 0.13043814897537231, + "r2": 0.6294513940811157, + "score": -0.23553038961691936 + }, + "test": { + "rmse": 0.2411268296110548, + "mae": 0.15842990577220917, + "r2": 0.6026889085769653, + "score": -0.2411268296110548 + } + }, + "time": "0:02:49.448996", + "diversity": { + "train": { + "mean_centered_corr": -0.018503453582525253, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04646626114845276 + }, + "val": { + "mean_centered_corr": -0.018042705953121185, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06875067204236984 + }, + "test": { + "mean_centered_corr": -0.016347019001841545, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09534233063459396 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1a1c1c437 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/sweeps/sberbank-housing/rla_rank8_noise1e-05-greedy-heads-evaluation/0/report.json @@ -0,0 +1,210 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 8, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 6702952, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 8 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2187223246642418, + "mae": 0.1176944449543953, + "r2": 0.6943670511245728, + "score": -0.2187223246642418 + }, + "val": { + "rmse": 0.23346652550110514, + "mae": 0.1266631931066513, + "r2": 0.6359169483184814, + "score": -0.23346652550110514 + }, + "test": { + "rmse": 0.24465339979644968, + "mae": 0.16457043588161469, + "r2": 0.5909823179244995, + "score": -0.24465339979644968 + } + }, + "time": "0:02:49.448996", + "diversity": { + "train": { + "mean_centered_corr": -0.018503453582525253, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04646626114845276 + }, + "val": { + "mean_centered_corr": -0.018042705953121185, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06875067204236984 + }, + "test": { + "mean_centered_corr": -0.016347019001841545, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09534233063459396 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 18, + 19, + 4, + 27, + 21, + 12, + 24, + 1 + ], + "head_selection_time": "0:00:00.130591" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/selected_integrated_configs.csv b/tabm_integrated/paper/exp/selected_integrated_configs.csv new file mode 100644 index 000000000..474f775f8 --- /dev/null +++ b/tabm_integrated/paper/exp/selected_integrated_configs.csv @@ -0,0 +1,31 @@ +dataset,final_variant,source_variant,source_wave,inference_mode,validation_metric,validation_score,source_config_path +sberbank-housing,baseline_plr,baseline_plr,smoke,mean,,,exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0.toml +sberbank-housing,best_rla_only,rla_rank2_noise0.001,sweeps,greedy-heads,0.2328613038082034,-0.2328613038082034,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0.toml +sberbank-housing,best_esam_only,esam_rho0.0025,sweeps,greedy-heads,0.23295802298295287,-0.23295802298295287,exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0.toml +sberbank-housing,best_mfb_only,mfb_keep0.8,sweeps,greedy-heads,0.23237596587359466,-0.23237596587359466,exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0.toml +sberbank-housing,best_cf_fisd_only,cf_fisd_only,smoke,greedy-heads,0.23327114512588593,-0.23327114512588593,exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0.toml +sberbank-housing,best_combined,mfb_cf_fisd,sweeps,greedy-heads,0.23320900639668657,-0.23320900639668657,exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0.toml +ecom-offers,baseline_plr,baseline_plr,smoke,mean,,,exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0.toml +ecom-offers,best_rla_only,rla_rank2_noise0.0001,sweeps,best-head,0.6396541901648656,0.6396541901648656,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0.toml +ecom-offers,best_esam_only,esam_rho0.001,sweeps,greedy-heads,0.641845068859757,0.641845068859757,exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0.toml +ecom-offers,best_mfb_only,mfb_keep0.7,sweeps,best-head,0.6547032702847417,0.6547032702847417,exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0.toml +ecom-offers,best_cf_fisd_only,cf_fisd_lambda0.05,sweeps,greedy-heads,0.6419149054445072,0.6419149054445072,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0.toml +ecom-offers,best_combined,mfb_cf_fisd,sweeps,greedy-heads,0.6506783465179156,0.6506783465179156,exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0.toml +homesite-insurance,baseline_plr,baseline_plr,smoke,mean,,,exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0.toml +homesite-insurance,best_rla_only,rla_rank4_noise1e-05,sweeps,greedy-heads,0.9591329763748824,0.9591329763748824,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0.toml +homesite-insurance,best_esam_only,esam_only,smoke,greedy-heads,0.958991133281215,0.958991133281215,exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0.toml +homesite-insurance,best_mfb_only,mfb_keep0.95,sweeps,greedy-heads,0.9592957095194116,0.9592957095194116,exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0.toml +homesite-insurance,best_cf_fisd_only,cf_fisd_only,smoke,greedy-heads,0.9594844321379927,0.9594844321379927,exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0.toml +homesite-insurance,best_combined,mfb_cf_fisd,sweeps,greedy-heads,0.9597570226838343,0.9597570226838343,exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0.toml +cooking-time,baseline_plr,baseline_plr,smoke,mean,,,exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0.toml +cooking-time,best_rla_only,rla_rank4_noise0.001,sweeps,greedy-heads,0.4624107384246987,-0.4624107384246987,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0.toml +cooking-time,best_esam_only,esam_rho0.0025,sweeps,greedy-heads,0.46246732196836127,-0.46246732196836127,exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0.toml +cooking-time,best_mfb_only,mfb_keep0.8,sweeps,greedy-heads,0.4624822561623469,-0.4624822561623469,exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0.toml +cooking-time,best_cf_fisd_only,cf_fisd_lambda0.05,sweeps,greedy-heads,0.4628482268025946,-0.4628482268025946,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0.toml +cooking-time,best_combined,rla_esam,sweeps,greedy-heads,0.4624234187671886,-0.4624234187671886,exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0.toml +delivery-eta,baseline_plr,baseline_plr,smoke,mean,,,exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0.toml +delivery-eta,best_rla_only,rla_rank2_noise0.001,sweeps,greedy-heads,0.5552720495549479,-0.5552720495549479,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0.toml +delivery-eta,best_esam_only,esam_only,smoke,greedy-heads,0.5551513290724005,-0.5551513290724005,exp/integrated/smoke/delivery-eta/esam_only-evaluation/0.toml +delivery-eta,best_mfb_only,mfb_keep0.7,sweeps,greedy-heads,0.5551008108444078,-0.5551008108444078,exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0.toml +delivery-eta,best_cf_fisd_only,cf_fisd_lambda0.05,sweeps,greedy-heads,0.5556691040453957,-0.5556691040453957,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0.toml +delivery-eta,best_combined,rla_esam,sweeps,greedy-heads,0.5552988041877751,-0.5552988041877751,exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0.toml diff --git a/tabm_integrated/paper/lib/__init__.py b/tabm_integrated/paper/lib/__init__.py new file mode 100644 index 000000000..f3e5e9015 --- /dev/null +++ b/tabm_integrated/paper/lib/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations +from . import env as _env # Check the current working directory. +from .util import * # noqa: F403 + +del _env diff --git a/tabm_integrated/paper/lib/cf_fisd.py b/tabm_integrated/paper/lib/cf_fisd.py new file mode 100644 index 000000000..811a725f2 --- /dev/null +++ b/tabm_integrated/paper/lib/cf_fisd.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Mapping, Sequence + +import numpy as np +import torch +import torch.nn.functional as F +from torch import Tensor + +VARIANTS = ('softmax', 'l1norm', 'raw') +DEFAULT_TEACHER_NAMES: tuple[str, ...] = ('xgb', 'lgbm', 'cat') + + +def chunk_aggregate_r1(r1: Tensor, d_features: Sequence[int]) -> Tensor: + if r1.dim() != 2: + raise ValueError(f'r1 must be 2D, got shape {tuple(r1.shape)}') + expected = int(sum(d_features)) + if r1.shape[-1] != expected: + raise ValueError( + f'r1 last dim {r1.shape[-1]} != sum(d_features)={expected}' + ) + abs_r = r1.abs() + chunks = abs_r.split(list(d_features), dim=-1) + return torch.stack([c.sum(dim=-1) for c in chunks], dim=-1) + + +def cf_fisd_loss( + r1: Tensor, + teacher_importances: Mapping[str, Tensor], + member_groups: Mapping[str, Sequence[int]], + variant: str, + d_features: Sequence[int], +) -> Tensor: + if variant not in VARIANTS: + raise ValueError(f'unknown variant: {variant!r}; expected one of {VARIANTS}') + if not member_groups: + return r1.new_zeros(()) + + r1_mag = chunk_aggregate_r1(r1, d_features) + losses: list[Tensor] = [] + for teacher_name, member_ids in member_groups.items(): + if not member_ids: + continue + if teacher_name not in teacher_importances: + raise KeyError(f'teacher_importances missing {teacher_name!r}') + t_imp = teacher_importances[teacher_name].to(r1.device, dtype=r1_mag.dtype) + for m in member_ids: + r_m = r1_mag[int(m)] + if variant == 'softmax': + pred = F.softmax(r_m, dim=-1) + tgt = F.softmax(t_imp, dim=-1) + elif variant == 'l1norm': + pred = r_m / (r_m.sum() + 1e-8) + tgt = t_imp / (t_imp.sum() + 1e-8) + else: + tgt = t_imp / (t_imp.sum() + 1e-8) * r_m.sum().detach() + pred = r_m + losses.append(F.mse_loss(pred, tgt, reduction='mean')) + if not losses: + return r1.new_zeros(()) + return torch.stack(losses).mean() + + +def alignment_cosine( + r1: Tensor, + teacher_imp: Tensor, + member_ids: Sequence[int], + d_features: Sequence[int], +) -> Tensor: + r1_soft = F.softmax(chunk_aggregate_r1(r1, d_features), dim=-1) + t_soft = F.softmax(teacher_imp.to(r1.device, dtype=r1_soft.dtype), dim=-1) + ids = list(member_ids) + return F.cosine_similarity( + r1_soft[ids], t_soft.unsqueeze(0).expand(len(ids), -1), dim=-1 + ) + + +def load_teacher_importances( + teacher_dir: str | Path, + dataset_name: str, + n_features: int, + teacher_names: Sequence[str] = DEFAULT_TEACHER_NAMES, +) -> dict[str, Tensor]: + out: dict[str, Tensor] = {} + root = Path(teacher_dir) + for name in teacher_names: + f = root / f'{name}.npy' + if not f.exists(): + f_alt = root / f'{dataset_name}_{name}.npy' + if f_alt.exists(): + f = f_alt + else: + raise FileNotFoundError(f) + v = np.load(f).astype(np.float32) + if v.shape != (n_features,): + raise ValueError( + f'{f}: expected shape ({n_features},), got {v.shape}' + ) + v = np.clip(v, 0.0, None) + s = float(v.sum()) + if s > 0.0: + v = v / s + else: + v = np.full_like(v, 1.0 / n_features) + out[name] = torch.from_numpy(v) + return out + + +def default_member_groups( + k: int, + teacher_names: Sequence[str] = DEFAULT_TEACHER_NAMES, +) -> dict[str, list[int]]: + n = len(teacher_names) + if n == 0: + raise ValueError('teacher_names must not be empty') + base = k // n + extra = k - base * n + groups: dict[str, list[int]] = {} + cursor = 0 + for i, name in enumerate(teacher_names): + size = base + (1 if i < extra else 0) + groups[name] = list(range(cursor, cursor + size)) + cursor += size + return groups diff --git a/tabm_integrated/paper/lib/data.py b/tabm_integrated/paper/lib/data.py new file mode 100644 index 000000000..22562fb44 --- /dev/null +++ b/tabm_integrated/paper/lib/data.py @@ -0,0 +1,397 @@ +from __future__ import annotations +import enum +import hashlib +import json +import pickle +from collections.abc import Iterable +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Generic, TypeVar, cast + +import numpy as np +import sklearn.preprocessing +import torch +from loguru import logger +from torch import Tensor + +from . import env +from .metrics import calculate_metrics as calculate_metrics_ +from .util import DataKey, PartKey, PredictionType, Score, TaskType + +_SCORE_SHOULD_BE_MAXIMIZED = { + Score.ACCURACY: True, + Score.CROSS_ENTROPY: False, + Score.MAE: False, + Score.R2: True, + Score.RMSE: False, + Score.ROC_AUC: True, +} + + +@dataclass(frozen=True) +class Task: + labels: dict[PartKey, np.ndarray] + type_: TaskType + score: Score + + @classmethod + def from_dir(cls, path: str | Path) -> 'Task': + path = Path(path).resolve() + info = json.loads(path.joinpath('info.json').read_text()) + task_type = TaskType(info['task_type']) + score = info.get('score') + if score is None: + score = { + TaskType.BINCLASS: Score.ACCURACY, + TaskType.MULTICLASS: Score.ACCURACY, + TaskType.REGRESSION: Score.RMSE, + }[task_type] + else: + score = Score(score) + return Task( + { + part: np.load(path / f'Y_{part}.npy') + for part in ['train', 'val', 'test'] + }, + task_type, + score, + ) + + def __post_init__(self): + assert isinstance(self.type_, TaskType) + assert isinstance(self.score, Score) + if self.is_regression: + assert all( + value.dtype in (np.dtype('float32'), np.dtype('float64')) + for value in self.labels.values() + ), 'Regression labels must have dtype=float32' + for key in self.labels: + self.labels[key] = self.labels[key].astype('float32') + + @property + def is_regression(self) -> bool: + return self.type_ == TaskType.REGRESSION + + @property + def is_binclass(self) -> bool: + return self.type_ == TaskType.BINCLASS + + @property + def is_multiclass(self) -> bool: + return self.type_ == TaskType.MULTICLASS + + @property + def is_classification(self) -> bool: + return self.is_binclass or self.is_multiclass + + def compute_n_classes(self) -> int: + assert self.is_binclass or self.is_classification + return len(np.unique(self.labels['train'])) + + def try_compute_n_classes(self) -> None | int: + return None if self.is_regression else self.compute_n_classes() + + def calculate_metrics( + self, + predictions: dict[PartKey, np.ndarray], + prediction_type: str | PredictionType, + ) -> dict[PartKey, Any]: + metrics = { + part: calculate_metrics_( + self.labels[part], predictions[part], self.type_, prediction_type + ) + for part in predictions + } + for part_metrics in metrics.values(): + part_metrics['score'] = ( + 1.0 if _SCORE_SHOULD_BE_MAXIMIZED[self.score] else -1.0 + ) * part_metrics[self.score.value] + return metrics # type: ignore[code] + + +def load_data(path: str | Path) -> dict[DataKey, dict[PartKey, np.ndarray]]: + path = Path(path).resolve() + return { # type: ignore[code] + key.lower(): { + part: np.load(path / f'{key}_{part}.npy', allow_pickle=True) + for part in ['train', 'val', 'test'] + } + for key in ['X_num', 'X_bin', 'X_cat', 'Y'] + if path.joinpath(f'{key}_train.npy').exists() + } + + +T = TypeVar('T', np.ndarray, Tensor) + + +@dataclass +class Dataset(Generic[T]): + """Dataset = Data + Task + simple methods for convenience. + + The task is stored separately to ensure that the original labels never change. + """ + + data: dict[DataKey, dict[PartKey, T]] + task: Task + + @classmethod + def from_dir(cls, path: str | Path) -> 'Dataset[np.ndarray]': + return Dataset(load_data(path), Task.from_dir(path)) + + def __post_init__(self): + """Validate the data.""" + data = self.data + is_numpy = self._is_numpy() + + # >>> Check data types. + for key, allowed_dtypes in { + 'x_num': [np.dtype('float32')] if is_numpy else [torch.float32], + 'x_bin': [np.dtype('float32')] if is_numpy else [torch.float32], + 'x_cat': [] if is_numpy else [torch.int64], + 'y': ( + [np.dtype('float32'), np.dtype('float64'), np.dtype('int64')] + if is_numpy + else [torch.float32, torch.int64] + ), + }.items(): + if key in data: + for part, value in data[key].items(): + if key == 'x_cat' and is_numpy: + assert value.dtype in ( + np.dtype('int32'), + np.dtype('int64'), + ) or isinstance( + value.dtype, + np.dtypes.StrDType, # type: ignore[code] + ) + else: + assert value.dtype in allowed_dtypes, ( + f'The value data["{key}"]["{part}"] has dtype' + f' {value.dtype}, but it must be one of {allowed_dtypes}' + ) + + # >>> Fix data types. + if self.task.is_regression: + for key in data['y']: + data['y'][key] = ( + data['y'][key].astype('float32') + if self._is_numpy() + else data['y'][key].to(torch.float32) + ) + if 'x_cat' in data and data['x_cat']['train'].dtype == np.dtype('int32'): + for key in data['x_cat']: + data['x_cat'][key] = data['x_cat'][key].astype('int64') + + # >>> Check nans. + isnan = np.isnan if is_numpy else torch.isnan + for key in ['x_bin']: + if key in data: # type: ignore[code] + for part, value in data[key].items(): + assert not isnan( + value # type: ignore[code] + ).any(), f'data[\"{key}\"][\"{part}\"] contains nans' + for part, value in data['y'].items(): + assert not isnan(value).any(), f'data["y"]["{part}"] contains nans' # type: ignore[code] + + def _is_numpy(self) -> bool: + return isinstance(self.data['y']['train'], np.ndarray) + + def __contains__(self, key: DataKey) -> bool: + return key in self.data + + def __getitem__(self, key: DataKey) -> dict[PartKey, T]: + return self.data[key] + + def __setitem__(self, key: DataKey, value: dict[PartKey, T]) -> None: + self.data[key] = value + + @property + def n_num_features(self) -> int: + return self.data['x_num']['train'].shape[1] if 'x_num' in self.data else 0 + + @property + def n_bin_features(self) -> int: + return self.data['x_bin']['train'].shape[1] if 'x_bin' in self.data else 0 + + @property + def n_cat_features(self) -> int: + return self.data['x_cat']['train'].shape[1] if 'x_cat' in self.data else 0 + + @property + def n_features(self) -> int: + return self.n_num_features + self.n_bin_features + self.n_cat_features + + def size(self, part: None | PartKey) -> int: + return ( + sum(map(len, self.data['y'].values())) + if part is None + else len(self.data['y'][part]) + ) + + def parts(self) -> Iterable[PartKey]: + return self.data['y'].keys() + + def compute_cat_cardinalities(self) -> list[int]: + x_cat = self.data.get('x_cat') + if x_cat is None: + return [] + unique = np.unique if self._is_numpy() else torch.unique + return ( + [] + if x_cat is None + else [len(unique(column)) for column in x_cat['train'].T] + ) + + def to_torch(self, device: None | str | torch.device) -> 'Dataset[Tensor]': + return Dataset( + { + key: { + part: torch.as_tensor(value).to(device) + for part, value in self.data[key].items() + } + for key in self.data + }, + self.task, + ) + + +class NumPolicy(enum.Enum): + STANDARD = 'standard' + NOISY_QUANTILE = 'noisy-quantile' + + +def transform_num( + X_num: dict[PartKey, np.ndarray], policy: None | str | NumPolicy, seed: None | int +) -> dict[PartKey, np.ndarray]: + if policy is not None: + policy = NumPolicy(policy) + X_num_train = X_num['train'] + if policy == NumPolicy.STANDARD: + normalizer = sklearn.preprocessing.StandardScaler() + elif policy == NumPolicy.NOISY_QUANTILE: + normalizer = sklearn.preprocessing.QuantileTransformer( + n_quantiles=max(min(X_num['train'].shape[0] // 30, 1000), 10), + output_distribution='normal', + subsample=1_000_000_000, + random_state=seed, + ) + assert seed is not None + X_num_train = X_num_train + np.random.RandomState(seed).normal( + 0.0, 1e-5, X_num_train.shape + ).astype(X_num_train.dtype) + else: + raise ValueError(f'Unknown policy={policy}') + + normalizer.fit(X_num_train) + X_num = {k: normalizer.transform(v) for k, v in X_num.items()} # type: ignore[code] + + # NOTE + # (This is not a good way to process NaNs) + # This is a quick hack to stop failing on some datasets because of NaNs. + # NaNs are replaced with zeros (zero is the mean value for all features after + # the conventional preprocessing techniques). + X_num = {k: np.nan_to_num(v) for k, v in X_num.items()} + + # Remove columns with one constant value. + mask = np.array([len(np.unique(x)) > 1 for x in X_num['train'].T]) + X_num = {k: v[:, mask] for k, v in X_num.items()} + + X_num = {k: v.astype(np.float32) for k, v in X_num.items()} + return X_num + + +class CatPolicy(enum.Enum): + ORDINAL = 'ordinal' + ONE_HOT = 'one-hot' + + +def transform_cat( + X_cat: dict[PartKey, np.ndarray], policy: None | str | CatPolicy +) -> dict[PartKey, np.ndarray]: + if policy is None: + return X_cat + + policy = CatPolicy(policy) + + # The first step is always the ordinal encoding, + # even for the one-hot encoding. + unknown_value = np.iinfo('int64').max - 3 + encoder = sklearn.preprocessing.OrdinalEncoder( + handle_unknown='use_encoded_value', # type: ignore[code] + unknown_value=unknown_value, # type: ignore[code] + dtype='int64', # type: ignore[code] + ).fit(X_cat['train']) + X_cat = {k: encoder.transform(v) for k, v in X_cat.items()} + max_values = X_cat['train'].max(axis=0) + for part in ['val', 'test']: + part = cast(PartKey, part) + for column_idx in range(X_cat[part].shape[1]): + X_cat[part][X_cat[part][:, column_idx] == unknown_value, column_idx] = ( + max_values[column_idx] + 1 + ) + + if policy == CatPolicy.ORDINAL: + return X_cat + elif policy == CatPolicy.ONE_HOT: + encoder = sklearn.preprocessing.OneHotEncoder( + handle_unknown='ignore', + sparse=False, + dtype=np.float32, # type: ignore[code] + ) + encoder.fit(X_cat['train']) + return {k: cast(np.ndarray, encoder.transform(v)) for k, v in X_cat.items()} + else: + raise ValueError(f'Unknown policy={policy}') + + +@dataclass(frozen=True, kw_only=True) +class RegressionLabelStats: + mean: float + std: float + + +def standardize_labels( + y: dict[PartKey, np.ndarray], +) -> tuple[dict[PartKey, np.ndarray], RegressionLabelStats]: + assert y['train'].dtype == np.dtype('float32') + mean = float(y['train'].mean()) + std = float(y['train'].std()) + return {k: (v - mean) / std for k, v in y.items()}, RegressionLabelStats( + mean=mean, std=std + ) + + +def build_dataset( + path: str | Path, + *, + num_policy: None | str | NumPolicy = None, + cat_policy: None | str | CatPolicy = None, + seed: int = 0, + cache: bool = False, +) -> Dataset[np.ndarray]: + path = Path(path).resolve() + if cache: + args = locals() + args.pop('cache') + args.pop('path') + cache_path = env.get_cache_dir() / ( + f'build_dataset__{path.name}__{hashlib.md5(str(args).encode("utf-8")).hexdigest()}.pickle' + ) + if cache_path.exists(): + cached_args, cached_value = pickle.loads(cache_path.read_bytes()) + assert args == cached_args, f'Hash collision for {cache_path}' + logger.info(f'Using cached dataset: {cache_path.name}') + return cached_value + else: + args = None + cache_path = None + + dataset = Dataset.from_dir(path) + if 'x_num' in dataset.data: + dataset['x_num'] = transform_num(dataset['x_num'], num_policy, seed) + if 'x_cat' in dataset.data: + dataset['x_cat'] = transform_cat(dataset['x_cat'], cat_policy) + + if cache_path is not None: + cache_path.write_bytes(pickle.dumps((args, dataset))) + return dataset diff --git a/tabm_integrated/paper/lib/deep.py b/tabm_integrated/paper/lib/deep.py new file mode 100644 index 000000000..cf90f20b5 --- /dev/null +++ b/tabm_integrated/paper/lib/deep.py @@ -0,0 +1,651 @@ +import itertools +from typing import Any, Literal + +import rtdl_num_embeddings +import rtdl_revisiting_models +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch import Tensor +from torch.nn import Parameter + + +# ====================================================================================== +# Initialization +# ====================================================================================== +def init_rsqrt_uniform_(x: Tensor, d: int) -> Tensor: + assert d > 0 + d_rsqrt = d**-0.5 + return nn.init.uniform_(x, -d_rsqrt, d_rsqrt) + + +@torch.inference_mode() +def init_random_signs_(x: Tensor) -> Tensor: + return x.bernoulli_(0.5).mul_(2).add_(-1) + + +# ====================================================================================== +# Modules +# ====================================================================================== +class Identity(nn.Module): + def __init__(self, *args, **kwargs) -> None: + super().__init__() + + def forward(self, x: Tensor) -> Tensor: + return x + + +class NLinear(nn.Module): + """A stack of N linear layers. Each layer is applied to its own part of the input. + + **Shape** + + - Input: ``(B, N, in_features)`` + - Output: ``(B, N, out_features)`` + + The i-th linear layer is applied to the i-th matrix of the shape (B, in_features). + + Technically, this is a simplified version of delu.nn.NLinear: + https://yura52.github.io/delu/stable/api/generated/delu.nn.NLinear.html. + The difference is that this layer supports only 3D inputs + with exactly one batch dimension. By contrast, delu.nn.NLinear supports + any number of batch dimensions. + """ + + def __init__( + self, n: int, in_features: int, out_features: int, bias: bool = True + ) -> None: + super().__init__() + self.weight = Parameter(torch.empty(n, in_features, out_features)) + self.bias = Parameter(torch.empty(n, out_features)) if bias else None + self.reset_parameters() + + def reset_parameters(self): + d = self.weight.shape[-2] + init_rsqrt_uniform_(self.weight, d) + if self.bias is not None: + init_rsqrt_uniform_(self.bias, d) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + assert x.ndim == 3 + assert x.shape[-(self.weight.ndim - 1) :] == self.weight.shape[:-1] + + x = x.transpose(0, 1) + x = x @ self.weight + x = x.transpose(0, 1) + if self.bias is not None: + x = x + self.bias + return x + + +class PiecewiseLinearEmbeddings(rtdl_num_embeddings.PiecewiseLinearEmbeddings): + """ + This class simply adds the default values for `activation` and `version`. + """ + + def __init__( + self, + *args, + activation: bool = False, + version: None | Literal['A', 'B'] = 'B', + **kwargs, + ) -> None: + super().__init__(*args, **kwargs, activation=activation, version=version) + + +class OneHotEncoding0d(nn.Module): + # Input: (*, n_cat_features=len(cardinalities)) + # Output: (*, sum(cardinalities)) + + def __init__(self, cardinalities: list[int]) -> None: + super().__init__() + self._cardinalities = cardinalities + + def forward(self, x: Tensor) -> Tensor: + assert x.ndim >= 1 + assert x.shape[-1] == len(self._cardinalities) + + return torch.cat( + [ + # NOTE + # This is a quick hack to support out-of-vocabulary categories. + # + # Recall that lib.data.transform_cat encodes categorical features + # as follows: + # - In-vocabulary values receive indices from `range(cardinality)`. + # - All out-of-vocabulary values (i.e. new categories in validation + # and test data that are not presented in the training data) + # receive the index `cardinality`. + # + # As such, the line below will produce the standard one-hot encoding for + # known categories, and the all-zeros encoding for unknown categories. + # This may not be the best approach to deal with unknown values, + # but should be enough for our purposes. + F.one_hot(x[..., i], cardinality + 1)[..., :-1] + for i, cardinality in enumerate(self._cardinalities) + ], + -1, + ) + + +class ScaleEnsemble(nn.Module): + def __init__( + self, + k: int, + d: int, + *, + init: Literal['ones', 'normal', 'random-signs'], + ) -> None: + super().__init__() + self.weight = nn.Parameter(torch.empty(k, d)) + self._weight_init = init + self.reset_parameters() + + def reset_parameters(self) -> None: + if self._weight_init == 'ones': + nn.init.ones_(self.weight) + elif self._weight_init == 'normal': + nn.init.normal_(self.weight) + elif self._weight_init == 'random-signs': + init_random_signs_(self.weight) + else: + raise ValueError(f'Unknown weight_init: {self._weight_init}') + + def forward(self, x: Tensor) -> Tensor: + assert x.ndim >= 2 + return x * self.weight + + +class LinearEfficientEnsemble(nn.Module): + """ + This layer is a more configurable version of the "BatchEnsemble" layer + from the paper + "BatchEnsemble: An Alternative Approach to Efficient Ensemble and Lifelong Learning" + (link: https://arxiv.org/abs/2002.06715). + + First, this layer allows to select only some of the "ensembled" parts: + - the input scaling (r_i in the BatchEnsemble paper) + - the output scaling (s_i in the BatchEnsemble paper) + - the output bias (not mentioned in the BatchEnsemble paper, + but is presented in public implementations) + + Second, the initialization of the scaling weights is configurable + through the `scaling_init` argument. + + NOTE + The term "adapter" is used in the TabM paper only to tell the story. + The original BatchEnsemble paper does NOT use this term. So this class also + avoids the term "adapter". + """ + + r: None | Tensor + s: None | Tensor + bias: None | Tensor + + def __init__( + self, + in_features: int, + out_features: int, + bias: bool = True, + *, + k: int, + ensemble_scaling_in: bool, + ensemble_scaling_out: bool, + ensemble_bias: bool, + scaling_init: Literal['ones', 'random-signs'], + ): + assert k > 0 + if ensemble_bias: + assert bias + super().__init__() + + self.weight = nn.Parameter(torch.empty(out_features, in_features)) + self.register_parameter( + 'r', + ( + nn.Parameter(torch.empty(k, in_features)) + if ensemble_scaling_in + else None + ), # type: ignore[code] + ) + self.register_parameter( + 's', + ( + nn.Parameter(torch.empty(k, out_features)) + if ensemble_scaling_out + else None + ), # type: ignore[code] + ) + self.register_parameter( + 'bias', + ( + nn.Parameter(torch.empty(out_features)) # type: ignore[code] + if bias and not ensemble_bias + else nn.Parameter(torch.empty(k, out_features)) + if ensemble_bias + else None + ), + ) + + self.in_features = in_features + self.out_features = out_features + self.k = k + self.scaling_init = scaling_init + + self.reset_parameters() + + def reset_parameters(self): + init_rsqrt_uniform_(self.weight, self.in_features) + scaling_init_fn = {'ones': nn.init.ones_, 'random-signs': init_random_signs_}[ + self.scaling_init + ] + if self.r is not None: + scaling_init_fn(self.r) + if self.s is not None: + scaling_init_fn(self.s) + if self.bias is not None: + bias_init = torch.empty( + # NOTE: the shape of bias_init is (out_features,) not (k, out_features). + # It means that all biases have the same initialization. + # This is similar to having one shared bias plus + # k zero-initialized non-shared biases. + self.out_features, + dtype=self.weight.dtype, + device=self.weight.device, + ) + bias_init = init_rsqrt_uniform_(bias_init, self.in_features) + with torch.inference_mode(): + self.bias.copy_(bias_init) + + def forward(self, x: Tensor) -> Tensor: + # x.shape == (B, K, D) + assert x.ndim == 3 + + # >>> The equation (5) from the BatchEnsemble paper (arXiv v2). + if self.r is not None: + x = x * self.r + x = x @ self.weight.T + if self.s is not None: + x = x * self.s + # <<< + + if self.bias is not None: + x = x + self.bias + return x + + +class LinearEfficientEnsembleRankR(nn.Module): + """Rank-r generalization of `LinearEfficientEnsemble`. + + Replaces the per-member rank-1 outer-product adapter `s_i r_i^T` + with a rank-r factorization `S_i R_i^T`, where + `R_i in R^{d_in x rank}` and `S_i in R^{d_out x rank}`. + + The per-member weight is `W_i = W ⊙ (S_i R_i^T)`. Equivalently, + the rank-r forward is `r` parallel rank-1 paths sharing `W`, + summed before the bias is added: + + y_i = sum_{j=1..rank} s_{i,j} ⊙ (W (r_{i,j} ⊙ x)) + b_i + + At ``rank == 1`` and ``additive == False`` this reduces exactly to + `LinearEfficientEnsemble` with both ``ensemble_scaling_in`` and + ``ensemble_scaling_out`` set to True. + + The ``additive`` flag switches the construction to an *additive* + LoRA-style correction `W_i = W + S_i R_i^T` for the defensive ablation + in Section 6.3 of the RLA spec. In this mode the per-member weight + starts equal to ``W`` because R/S are zero-initialized. + """ + + bias: None | Tensor + + def __init__( + self, + in_features: int, + out_features: int, + bias: bool = True, + *, + k: int, + rank: int = 1, + scaling_init: Literal['ones', 'random-signs'] = 'ones', + additive: bool = False, + init_mode: Literal['variance_preserving', 'base_preserving'] = 'variance_preserving', + base_preserve_noise: float = 1e-3, + ) -> None: + """ + ``init_mode`` selects how the rank-r adapter is initialised: + + * ``variance_preserving`` (default, original behaviour): every + column of R and S is filled with ``1/sqrt(r)`` so the sum over + rank paths preserves the variance of the rank-1 baseline. + For first-layer ``random-signs`` scaling, each R column gets + independent random signs scaled by ``1/sqrt(r)``. The forward + output is *not* bit-identical to the rank-1 baseline at rank>1 + (the per-path random signs interact differently). + + * ``base_preserving``: rank path 0 is initialised exactly as the + rank-1 baseline (R[:,:,0]=1 or random-signs, S[:,:,0]=1). + Extra rank paths j>=1 are initialised with S[:,:,j]=0 and + R[:,:,j]=tiny noise (std=``base_preserve_noise``). At init the + extra paths contribute zero to the forward output, so a + base-preserving rank-r RLA layer produces *bit-identical* + forward outputs to the baseline (rank-1) layer. After one + optimiser step S becomes non-zero and R picks up gradient. + """ + assert k > 0 + assert rank >= 1 + super().__init__() + + self.in_features = in_features + self.out_features = out_features + self.k = k + self.rank = rank + self.scaling_init = scaling_init + self.additive = additive + self.init_mode = init_mode + self.base_preserve_noise = base_preserve_noise + + self.weight = nn.Parameter(torch.empty(out_features, in_features)) + # R: (k, d_in, rank), S: (k, d_out, rank) + self.R = nn.Parameter(torch.empty(k, in_features, rank)) + self.S = nn.Parameter(torch.empty(k, out_features, rank)) + self.register_parameter( + 'bias', + nn.Parameter(torch.empty(k, out_features)) if bias else None, + ) + + self.reset_parameters() + + def reset_parameters(self) -> None: + init_rsqrt_uniform_(self.weight, self.in_features) + + if self.additive: + # Additive LoRA-style start at the baseline weight: zero S, normal R. + nn.init.zeros_(self.S) + init_rsqrt_uniform_(self.R, self.in_features) + elif self.init_mode == 'base_preserving': + # Path 0 reproduces the rank-1 baseline exactly. Extra paths + # are zero-gated on the S side (so they contribute nothing to + # the forward output at init) and receive tiny noise on the + # R side (so the next step gives S non-zero gradient). + # + # NOTE on RNG ordering: to keep the shared weight and bias + # element-identical to the baseline class under the same seed, + # the only RNG-consuming step inside the adapter init must + # match what the baseline does (ones-fill = no RNG). The + # extra-path noise *would* break that match, so we defer it + # until after the bias has been drawn (see below). + with torch.inference_mode(): + # Path 0 of S: deterministic 1.0 (matches baseline 's=ones'). + self.S.zero_() + self.S[:, :, 0].fill_(1.0) + # Path 0 of R: depends on scaling_init. + self.R.zero_() + if self.scaling_init == 'random-signs': + # Note: this draws RNG, just like the baseline class + # does for its own r=random-signs init, so equivalent + # baseline + base_preserving construction stays in sync. + sign_path = torch.empty_like(self.R[:, :, 0]) + init_random_signs_(sign_path) + self.R[:, :, 0].copy_(sign_path) + elif self.scaling_init == 'ones': + self.R[:, :, 0].fill_(1.0) + else: + raise ValueError(f'Unknown scaling_init: {self.scaling_init}') + elif self.init_mode == 'variance_preserving': + # Variance-preserving (original) form. Match the rank-1 + # baseline exactly only at rank=1. + scale = self.rank**-0.5 + if self.scaling_init == 'random-signs': + init_random_signs_(self.R) + self.R.data.mul_(scale) + with torch.inference_mode(): + self.S.fill_(scale) + elif self.scaling_init == 'ones': + with torch.inference_mode(): + self.R.fill_(scale) + self.S.fill_(scale) + else: + raise ValueError(f'Unknown scaling_init: {self.scaling_init}') + else: + raise ValueError(f'Unknown init_mode: {self.init_mode}') + + if self.bias is not None: + bias_init = torch.empty( + self.out_features, + dtype=self.weight.dtype, + device=self.weight.device, + ) + bias_init = init_rsqrt_uniform_(bias_init, self.in_features) + with torch.inference_mode(): + self.bias.copy_(bias_init) + + # Deferred extra-path noise for base_preserving. + # + # CRITICAL: this draw must NOT advance the global RNG, otherwise + # downstream layers in the same model will be initialised at a + # different RNG state than the baseline class would have used, + # breaking full-model bit-equivalence at rank 1 / rank > 1. + # We isolate the noise draw by save+restore of the global RNG + # state and use a separate Generator seeded deterministically + # from a hash of the parameter shape (so the noise pattern is + # reproducible across runs without consuming the main RNG). + if ( + self.init_mode == 'base_preserving' + and not self.additive + and self.rank > 1 + and self.base_preserve_noise > 0 + ): + # Deterministic per-layer seed: stable cross-process hash of + # the shape tuple via hashlib (Python's built-in hash() is + # process-salted by PYTHONHASHSEED, so identical configs would + # otherwise initialise the extra-path noise differently across + # processes; we want bitwise reproducibility). + import hashlib + shape_key = ( + f'rla_base_preserve_noise|' + f'{self.in_features}|{self.out_features}|{self.k}|{self.rank}' + ).encode('utf-8') + seed = ( + int.from_bytes(hashlib.sha256(shape_key).digest()[:4], 'big') + & 0x7FFFFFFF + ) + gen = torch.Generator(device=self.R.device) + gen.manual_seed(seed) + noise = torch.empty_like(self.R[:, :, 1:]) + noise.normal_(mean=0.0, std=self.base_preserve_noise, generator=gen) + with torch.inference_mode(): + self.R[:, :, 1:].copy_(noise) + + def forward(self, x: Tensor) -> Tensor: + # x.shape == (B, K, D_in) + assert x.ndim == 3 + # Backbone projection (shared across rank paths and members). + Wx = x @ self.weight.T # (B, K, D_out) + + if self.additive: + # W_i x = W x + S_i (R_i^T x). Sum the rank-r correction. + # x: (B,K,Din) R: (K,Din,r) -> z: (B,K,r) per member + z = torch.einsum('bki,kir->bkr', x, self.R) + # S: (K, Dout, r) -> correction: (B,K,Dout) + corr = torch.einsum('bkr,kor->bko', z, self.S) + y = Wx + corr + else: + # Multiplicative: y = sum_j S[:,:,j] ⊙ (W (R[:,:,j] ⊙ x)). + # Loop over rank (rank ≤ 8, GEMM-bound; loop overhead negligible). + y = x.new_zeros(x.shape[0], x.shape[1], self.out_features) + for j in range(self.rank): + # x_scaled: (B,K,Din) — element-wise scale by R[:,:,j] (K,Din) + x_scaled = x * self.R[:, :, j] + proj = x_scaled @ self.weight.T # (B,K,Dout) + y = y + proj * self.S[:, :, j] # broadcast (K,Dout) -> (B,K,Dout) + + if self.bias is not None: + y = y + self.bias + return y + + +def make_efficient_ensemble(module: nn.Module, EnsembleLayer, **kwargs) -> None: + """Replace linear layers with efficient ensembles of linear layers. + + NOTE + In the paper, there are no experiments with networks with normalization layers. + Perhaps, their trainable weights (the affine transformations) also need + "ensemblification" as in the paper about "FiLM-Ensemble". + Additional experiments are required to make conclusions. + """ + for name, submodule in list(module.named_children()): + if isinstance(submodule, nn.Linear): + module.add_module( + name, + EnsembleLayer( + in_features=submodule.in_features, + out_features=submodule.out_features, + bias=submodule.bias is not None, + **kwargs, + ), + ) + else: + make_efficient_ensemble(submodule, EnsembleLayer, **kwargs) + + +class MLP(nn.Module): + def __init__( + self, + *, + d_in: None | int = None, + d_out: None | int = None, + n_blocks: int, + d_block: int, + dropout: float, + activation: str = 'ReLU', + ) -> None: + super().__init__() + + d_first = d_block if d_in is None else d_in + self.blocks = nn.ModuleList( + [ + nn.Sequential( + nn.Linear(d_first if i == 0 else d_block, d_block), + getattr(nn, activation)(), + nn.Dropout(dropout), + ) + for i in range(n_blocks) + ] + ) + self.output = None if d_out is None else nn.Linear(d_block, d_out) + + def forward(self, x: Tensor) -> Tensor: + for block in self.blocks: + x = block(x) + if self.output is not None: + x = self.output(x) + return x + + +_CUSTOM_MODULES = { + # https://docs.python.org/3/library/stdtypes.html#definition.__name__ + CustomModule.__name__: CustomModule + for CustomModule in [ + rtdl_num_embeddings.LinearEmbeddings, + rtdl_num_embeddings.LinearReLUEmbeddings, + rtdl_num_embeddings.PeriodicEmbeddings, + PiecewiseLinearEmbeddings, + MLP, + ] +} + + +def make_module(type: str, *args, **kwargs) -> nn.Module: + Module = getattr(nn, type, None) + if Module is None: + Module = _CUSTOM_MODULES[type] + return Module(*args, **kwargs) + + +def get_n_parameters(m: nn.Module): + return sum(x.numel() for x in m.parameters() if x.requires_grad) + + +@torch.inference_mode() +def compute_parameter_stats(module: nn.Module) -> dict[str, dict[str, float]]: + stats = {'norm': {}, 'gradnorm': {}, 'gradratio': {}} + for name, parameter in module.named_parameters(): + stats['norm'][name] = parameter.norm().item() + if parameter.grad is not None: + stats['gradnorm'][name] = parameter.grad.norm().item() + # Avoid computing statistics for zero-initialized parameters. + if (parameter.abs() > 1e-6).any(): + stats['gradratio'][name] = ( + (parameter.grad.abs() / parameter.abs().clamp_min_(1e-6)) + .mean() + .item() + ) + stats['norm']['model'] = ( + torch.cat([x.flatten() for x in module.parameters()]).norm().item() + ) + stats['gradnorm']['model'] = ( + torch.cat([x.grad.flatten() for x in module.parameters() if x.grad is not None]) + .norm() + .item() + ) + return stats + + +# ====================================================================================== +# Optimization +# ====================================================================================== +def default_zero_weight_decay_condition( + module_name: str, module: nn.Module, parameter_name: str, parameter: Parameter +): + from rtdl_num_embeddings import _Periodic + + del module_name, parameter + return parameter_name.endswith('bias') or isinstance( + module, + nn.BatchNorm1d + | nn.LayerNorm + | nn.InstanceNorm1d + | rtdl_revisiting_models.LinearEmbeddings + | rtdl_num_embeddings.LinearEmbeddings + | rtdl_num_embeddings.LinearReLUEmbeddings + | _Periodic, + ) + + +def make_parameter_groups( + module: nn.Module, + zero_weight_decay_condition=default_zero_weight_decay_condition, + custom_groups: None | list[dict[str, Any]] = None, +) -> list[dict[str, Any]]: + if custom_groups is None: + custom_groups = [] + custom_params = frozenset( + itertools.chain.from_iterable(group['params'] for group in custom_groups) + ) + assert len(custom_params) == sum( + len(group['params']) for group in custom_groups + ), 'Parameters in custom_groups must not intersect' + zero_wd_params = frozenset( + p + for mn, m in module.named_modules() + for pn, p in m.named_parameters() + if p not in custom_params and zero_weight_decay_condition(mn, m, pn, p) + ) + default_group = { + 'params': [ + p + for p in module.parameters() + if p not in custom_params and p not in zero_wd_params + ] + } + return [ + default_group, + {'params': list(zero_wd_params), 'weight_decay': 0.0}, + *custom_groups, + ] + + +def make_optimizer(type: str, **kwargs) -> torch.optim.Optimizer: + Optimizer = getattr(torch.optim, type) + return Optimizer(**kwargs) diff --git a/tabm_integrated/paper/lib/env.py b/tabm_integrated/paper/lib/env.py new file mode 100644 index 000000000..42550d4f4 --- /dev/null +++ b/tabm_integrated/paper/lib/env.py @@ -0,0 +1,28 @@ +from __future__ import annotations +# NOTE: this file must not import anything from lib. + +from pathlib import Path + +_PROJECT_DIR = Path.cwd() + +assert _PROJECT_DIR.joinpath( + 'pixi.toml' +).exists(), 'The script must be run from the `paper/` directory' + + +def get_project_dir() -> Path: + return _PROJECT_DIR + + +def get_cache_dir() -> Path: + path = get_project_dir() / 'cache' + path.mkdir(exist_ok=True) + return path + + +def get_data_dir() -> Path: + return get_project_dir() / 'data' + + +def get_exp_dir() -> Path: + return get_project_dir() / 'exp' diff --git a/tabm_integrated/paper/lib/metrics.py b/tabm_integrated/paper/lib/metrics.py new file mode 100644 index 000000000..5df03a12e --- /dev/null +++ b/tabm_integrated/paper/lib/metrics.py @@ -0,0 +1,66 @@ +from __future__ import annotations +from typing import Any, cast + +import numpy as np +import scipy.special +import sklearn.metrics + +from .util import PredictionType, TaskType + + +def _get_labels_and_probs( + prediction: np.ndarray, + task_type: TaskType, + prediction_type: PredictionType, +) -> tuple[np.ndarray, None | np.ndarray]: + """Obtain labels and probabilities from raw predictions.""" + assert task_type in (TaskType.BINCLASS, TaskType.MULTICLASS) + + if prediction_type == PredictionType.LABELS: + return prediction, None + elif prediction_type == PredictionType.PROBS: + probs = prediction + elif prediction_type == PredictionType.LOGITS: + probs = ( + scipy.special.expit(prediction) + if task_type == TaskType.BINCLASS + else scipy.special.softmax(prediction, axis=1) + ) + else: + raise ValueError(f'Unknown prediction type: {prediction_type}') + + assert probs is not None + labels = np.round(probs) if task_type == TaskType.BINCLASS else probs.argmax(axis=1) + return labels.astype(np.int64), probs + + +def calculate_metrics( + y_true: np.ndarray, + y_pred: np.ndarray, + task_type: str | TaskType, + prediction_type: str | PredictionType, +) -> dict[str, Any]: + task_type = TaskType(task_type) + prediction_type = PredictionType(prediction_type) + + if task_type == TaskType.REGRESSION: + assert prediction_type == PredictionType.LABELS + result = { + 'rmse': float(sklearn.metrics.mean_squared_error(y_true, y_pred) ** 0.5), + 'mae': float(sklearn.metrics.mean_absolute_error(y_true, y_pred)), + 'r2': float(sklearn.metrics.r2_score(y_true, y_pred)), + } + + else: + assert prediction_type is not None + labels, probs = _get_labels_and_probs(y_pred, task_type, prediction_type) + result = cast( + dict[str, Any], + sklearn.metrics.classification_report(y_true, labels, output_dict=True), + ) + if probs is not None: + result['cross-entropy'] = sklearn.metrics.log_loss(y_true, probs) + if task_type == TaskType.BINCLASS and probs is not None: + result['roc-auc'] = sklearn.metrics.roc_auc_score(y_true, probs) + + return result diff --git a/tabm_integrated/paper/lib/util.py b/tabm_integrated/paper/lib/util.py new file mode 100644 index 000000000..fbc8fc951 --- /dev/null +++ b/tabm_integrated/paper/lib/util.py @@ -0,0 +1,517 @@ +from __future__ import annotations +import argparse +import datetime +import enum +import importlib +import inspect +import json +import os +import shutil +import sys +import time +from collections.abc import Callable +from copy import deepcopy +from pathlib import Path +from pprint import pprint +from typing import Any, TypeVar, cast + +import numpy as np +import tomli +import tomli_w +from loguru import logger + +# NOTE +# This file must NOT import anything from lib except for `env`, +# because all other submodules are allowed to import `util`. +from . import env + +# The purpose of the following snippet is to optimize import times +# when slow-to-import modules are not needed. +_TORCH = None + + +def _torch(): + global _TORCH + if _TORCH is None: + import torch + + _TORCH = torch + return _TORCH + + +# ====================================================================================== +# Const +# ====================================================================================== +WORST_SCORE = -999999.0 + +# ====================================================================================== +# Types +# ====================================================================================== +KWArgs = dict[str, Any] +JSONDict = dict[str, Any] # Must be JSON-serializable. + +DataKey = str # 'x_num', 'x_bin', 'x_cat', 'y', ... +PartKey = str # 'train', 'val', 'test', ... + + +class TaskType(enum.Enum): + REGRESSION = 'regression' + BINCLASS = 'binclass' + MULTICLASS = 'multiclass' + + +class PredictionType(enum.Enum): + LABELS = 'labels' + PROBS = 'probs' + LOGITS = 'logits' + + +class Score(enum.Enum): + ACCURACY = 'accuracy' + CROSS_ENTROPY = 'cross-entropy' + MAE = 'mae' + R2 = 'r2' + RMSE = 'rmse' + ROC_AUC = 'roc-auc' + + +# ====================================================================================== +# Tools for the `main` function. +# +# The following utilities expect that the `main` function +# has one of the following signatures: +# +# 1. main(config, output = None, *, force: bool = False) -> None | JSONDict +# 2. main(config, output = None, *, force: bool = False, continue_: bool = False) -> None | JSONDict # noqa +# +# Notes: +# * `config` is a Python dictionary or a path to a config in the TOML format. +# * `output` is the output directory with all results of the run. +# If not provided, it it automatically inferred from the config path. +# * Setting `force=True` means removing the already existing output. +# * Setting `continue_=True` means continuing the execution from a checkpoint. +# * The return value is `report` -- a JSON-serializable Python dictionary +# with any information about the run. +# ====================================================================================== +T = TypeVar('T') + + +def check( + config, output: None | str | Path, *, config_type: type[T] = dict +) -> tuple[T, Path]: + """Load the config and infer the path to the output directory.""" + # >>> This is a snippet for the internal infrastructure, ignore it. + snapshot_dir = os.environ.get('SNAPSHOT_PATH') + if snapshot_dir and Path(snapshot_dir).joinpath('CHECKPOINTS_RESTORED').exists(): + assert inspect.stack()[1].frame.f_locals.get('continue_') + del snapshot_dir + # <<< + + # >>> Check paths. + if isinstance(config, str | Path): + # config is a path. + config = Path(config) + assert config.suffix == '.toml' + assert config.exists(), f'The config {config} does not exist' + if output is None: + # In this case, output is a directory located next to the config. + output = config.with_suffix('') + config = load_config(config) + else: + # config is already a dictionary. + assert ( + output is not None + ), 'If config is a dictionary, then the `output` directory must be provided.' + output = Path(output).resolve() + + # >>> Check the config. + if config_type is dict: + pass + elif ( + # If all conditions are True, config_type is assumed to be a TypedDict. + issubclass(config_type, dict) + and hasattr(config_type, '__required_keys__') + and hasattr(config_type, '__optional_keys__') + ): + # >>> Check the keys. + presented_keys = frozenset(config) + required_keys = config_type.__required_keys__ # type: ignore[code] + optional_keys = config_type.__optional_keys__ # type: ignore[code] + assert presented_keys >= required_keys, ( + 'The config is missing the following required keys:' + f' {", ".join(required_keys - presented_keys)}' + ) + assert set(config) <= (required_keys | optional_keys), ( + 'The config has unknown keys:' + f' {", ".join(presented_keys - required_keys - optional_keys)}' + ) + + return cast(T, config), output + + +def start(output: str | Path, *, continue_: bool = False, force: bool = False) -> bool: + """Create the output directory (if missing). + + Returns: + True if the caller should continue the execution. + False if the caller should immediately return. + """ + print_sep('=') + output = Path(output).resolve() + print(f'[>>>] {try_get_relative_path(output)} | {datetime.datetime.now()}') + print_sep('=') + + if output.exists(): + if force: + logger.warning('Removing the existing output') + time.sleep(2.0) # Keep the above message visible for some time. + shutil.rmtree(output) + output.mkdir() + return True + elif not continue_: + backup_output(output) + logger.warning('The output already exists!') + return False + elif output.joinpath('DONE').exists(): + backup_output(output) + logger.info('Already done!\n') + return False + else: + logger.info('Continuing with the existing output') + return True + else: + logger.info('Creating the output') + output.mkdir() + return True + + +def create_report(function, config) -> JSONDict: + return { + 'function': get_function_full_name(function), + 'gpus': get_gpu_names(), + 'config': jsonify(config), + } + + +def summarize(report: JSONDict) -> JSONDict: + """Summarize the key information from the report.""" + summary = {'function': report.get('function')} + + if 'best' in report: + # The gpus info is collected from the best report. + summary['best'] = summarize(report['best']) + elif 'gpus' in report: + summary['gpus'] = report['gpus'] + + for key in ['n_parameters', 'best_stage', 'best_epoch', 'tuning_time', 'trial_id']: + if key in report: + summary[key] = deepcopy(report[key]) + + metrics = report.get('metrics') + if metrics is not None and 'score' in next(iter(metrics.values())): + summary['scores'] = {part: metrics[part]['score'] for part in metrics} + + for key in ['n_completed_trials', 'time']: + if key in report: + summary[key] = deepcopy(report[key]) + + return summary + + +def finish(output: Path, report: JSONDict) -> None: + dump_report(output, report) + + # >>> A code block for the internal infrastructure, ignore it. + JSON_OUTPUT_FILE = os.environ.get('JSON_OUTPUT_FILE') + if JSON_OUTPUT_FILE: + try: + key = str(output.relative_to(env.get_project_dir())) + except ValueError: + pass + else: + json_output_path = Path(JSON_OUTPUT_FILE) + try: + json_data = json.loads(json_output_path.read_text()) + except (FileNotFoundError, json.decoder.JSONDecodeError): + json_data = {} + json_data[key] = load_report(output) + json_output_path.write_text(json.dumps(json_data, indent=4)) + shutil.copyfile( + json_output_path, + os.path.join(os.environ['SNAPSHOT_PATH'], 'json_output.json'), + ) + # <<< + + output.joinpath('DONE').touch() + backup_output(output) + print() + try: + print_summary(output) + except FileNotFoundError: + pass + print() + print_sep('=') + print(f'[<<<] {try_get_relative_path(output)} | {datetime.datetime.now()}') + print_sep('=') + + +def run(function: Callable[..., None | JSONDict]) -> None | JSONDict: + """Run CLI for the main function.""" + parser = argparse.ArgumentParser() + parser.add_argument('config') + parser.add_argument('--output') + parser.add_argument('--force', action='store_true') + if 'continue_' in inspect.signature(function).parameters: + parser.add_argument('--continue', action='store_true', dest='continue_') + + return function(**vars(parser.parse_args(sys.argv[1:]))) + + +# ====================================================================================== +# IO for the output directory +# ====================================================================================== +def load_config(output_or_config: str | Path) -> JSONDict: + return tomli.loads(Path(output_or_config).with_suffix('.toml').read_text()) + + +def dump_config( + output_or_config: str | Path, config: JSONDict, *, force: bool = False +) -> None: + config_path = Path(output_or_config).with_suffix('.toml') + if config_path.exists() and not force: + raise RuntimeError( + 'The following config already exists (pass force=True to overwrite it)' + f' {config_path}' + ) + config_path.write_text(tomli_w.dumps(config)) + + +def load_report(output: str | Path) -> JSONDict: + return json.loads(Path(output).joinpath('report.json').read_text()) + + +def dump_report(output: str | Path, report: JSONDict) -> None: + Path(output).joinpath('report.json').write_text(json.dumps(report, indent=4)) + + +def load_summary(output: str | Path) -> JSONDict: + return json.loads(Path(output).joinpath('summary.json').read_text()) + + +def print_summary(output: str | Path): + pprint(load_summary(output), sort_dicts=False, width=60) + + +def dump_summary(output: str | Path, summary: JSONDict) -> None: + Path(output).joinpath('summary.json').write_text(json.dumps(summary, indent=4)) + + +def load_predictions(output: str | Path) -> dict[PartKey, np.ndarray]: + x = np.load(Path(output) / 'predictions.npz') + return {key: x[key] for key in x} + + +def dump_predictions( + output: str | Path, predictions: dict[PartKey, np.ndarray] +) -> None: + np.savez(Path(output) / 'predictions.npz', **predictions) + + +def get_checkpoint_path(output: str | Path) -> Path: + return Path(output) / 'checkpoint.pt' + + +def load_checkpoint(output: str | Path, **kwargs) -> Any: + # PyTorch 2.6 changed torch.load default weights_only=True, which breaks + # loading our full training checkpoint objects unless explicitly disabled. + kwargs.setdefault('weights_only', False) + return _torch().load(get_checkpoint_path(output), **kwargs) + + +def dump_checkpoint(output: str | Path, checkpoint: JSONDict, **kwargs) -> None: + _torch().save(checkpoint, get_checkpoint_path(output), **kwargs) + + +# ====================================================================================== +# Printing +# ====================================================================================== +def print_sep(ch='-'): + print(ch * 100) + + +def print_config(config: dict) -> None: + print() + pprint(config, sort_dicts=False, width=100) + print() + + +def print_metrics(loss: float, metrics: dict) -> None: + print( + f'(val) {metrics["val"]["score"]:.3f}' + f' (test) {metrics["test"]["score"]:.3f}' + f' (loss) {loss:.5f}' + ) + + +def log_scores(metrics: dict) -> None: + logger.debug( + f'[val] {metrics["val"]["score"]:.4f} [test] {metrics["test"]["score"]:.4f}' + ) + + +# ====================================================================================== +# CUDA +# ====================================================================================== +def get_device(): # -> torch.device + torch = _torch() + return torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu') + + +def is_dataparallel_available() -> bool: + torch = _torch() + return ( + torch.cuda.is_available() + and torch.cuda.device_count() > 1 + and 'CUDA_VISIBLE_DEVICES' in os.environ + ) + + +def get_gpu_names() -> list[str]: + return [ + _torch().cuda.get_device_name(i) for i in range(_torch().cuda.device_count()) + ] + + +def is_oom_exception(err: RuntimeError) -> bool: + return isinstance(err, _torch().cuda.OutOfMemoryError) or any( + x in str(err) + for x in [ + 'CUDA out of memory', + 'CUBLAS_STATUS_ALLOC_FAILED', + 'CUDA error: out of memory', + ] + ) + + +# ====================================================================================== +# Other +# ====================================================================================== +def configure_libraries(): + torch = _torch() + torch.set_num_threads(1) + torch.backends.cuda.matmul.allow_tf32 = False # type: ignore[code] + torch.backends.cudnn.allow_tf32 = False # type: ignore[code] + torch.backends.cudnn.benchmark = False # type: ignore[code] + torch.backends.cudnn.deterministic = True # type: ignore[code] + + logger.remove() + logger.add(sys.stderr, format='{message}') + + +def try_get_relative_path(path: str | Path) -> Path: + path = Path(path).resolve() + project_dir = env.get_project_dir() + return path.relative_to(project_dir) if project_dir in path.parents else path + + +def jsonify(value): + if value is None or isinstance(value, bool | int | float | str | bytes): + return value + elif isinstance(value, list): + return [jsonify(x) for x in value] + elif isinstance(value, dict): + return {k: jsonify(v) for k, v in value.items()} + else: + return '' + + +def are_valid_predictions(predictions: dict) -> bool: + # predictions: dict[PartKey, np.ndarray] + assert all(isinstance(x, np.ndarray) for x in predictions.values()) + return all(np.isfinite(x).all() for x in predictions.values()) + + +def import_(qualname: str) -> Any: + """ + Examples: + + >>> import_('bin.model.main') + """ + try: + module, name = qualname.rsplit('.', 1) + return getattr(importlib.import_module(module), name) + except Exception as err: + raise ValueError(f'Cannot import "{qualname}"') from err + + +def get_function_full_name(function: Callable) -> str: + """ + Examples: + + >>> # In the script bin/model.py + >>> get_function_full_name(main) == 'bin.model.main' + + >>> # In the script a/b/c/foo.py + >>> assert get_function_full_name(main) == 'a.b.c.foo.main' + """ + module = inspect.getmodule(function) + assert module is not None, 'Failed to locate the module of the function.' + + module_path = getattr(module, '__file__', None) + assert module_path is not None, ( + 'Failed to locate the module of the function.' + ' This can happen if the code is running in a Jupyter notebook.' + ) + + module_path = Path(module_path).resolve() + project_dir = env.get_project_dir() + assert project_dir in module_path.parents, ( + 'The module of the function must be located within the project directory: ' + f' {project_dir}' + ) + + module_full_name = str( + module_path.relative_to(project_dir).with_suffix('') + ).replace('/', '.') + return f'{module_full_name}.{function.__name__}' + + +_LAST_SNAPSHOT_TIME = None + + +def backup_output(output: Path) -> None: + """A function for the internal infrastructure, ignore it.""" + backup_dir = os.environ.get('TMP_OUTPUT_PATH') + snapshot_dir = os.environ.get('SNAPSHOT_PATH') + if backup_dir is None: + assert snapshot_dir is None + return + assert snapshot_dir is not None + + try: + relative_output_dir = output.relative_to(env.get_project_dir()) + except ValueError: + return + + for dir_ in [backup_dir, snapshot_dir]: + new_output = dir_ / relative_output_dir + prev_backup_output = new_output.with_name(new_output.name + '_prev') + new_output.parent.mkdir(exist_ok=True, parents=True) + if new_output.exists(): + new_output.rename(prev_backup_output) + shutil.copytree(output, new_output) + # The case for evaluate.py which automatically creates configs. + if output.with_suffix('.toml').exists(): + shutil.copyfile( + output.with_suffix('.toml'), new_output.with_suffix('.toml') + ) + if prev_backup_output.exists(): + shutil.rmtree(prev_backup_output) + + global _LAST_SNAPSHOT_TIME + if _LAST_SNAPSHOT_TIME is None or time.time() - _LAST_SNAPSHOT_TIME > 10 * 60: + import nirvana_dl.snapshot # type: ignore[code] + + nirvana_dl.snapshot.dump_snapshot() + _LAST_SNAPSHOT_TIME = time.time() + print('The snapshot was saved!') diff --git a/tabm_integrated/paper/pixi.toml b/tabm_integrated/paper/pixi.toml new file mode 100644 index 000000000..c6373bf4d --- /dev/null +++ b/tabm_integrated/paper/pixi.toml @@ -0,0 +1,58 @@ +[workspace] +channels = ["nvidia", "conda-forge", "pytorch", "nodefaults"] +name = "tabm" +platforms = ["linux-64", "osx-arm64", "osx-64"] + +[dependencies] +loguru = "==0.7.2" +numpy = "==1.26.3" +optuna = "==3.3.0" +pandas = "==2.1.1" +polars = "==1.5.0" +python = "==3.10.12" +pytorch = { version = "==2.0.1", channel = "pytorch" } +scikit-learn = "==1.3.1" +scipy = "==1.11.3" +tensorboard = "==2.15.2" +tomli = "==2.0.1" +tomli-w = "==1.0.0" +tqdm = "==4.66.1" +typing-extensions = "==4.8.0" +# Development (results do not depend on these packages) +bokeh = "==3.3.4" +holoviews = "==1.18.3" +hvplot = "==0.10.0" +ipython = "==8.16.1" +ipywidgets = "==8.1.3" +jupyterlab = "==4.1.1" +mypy = "==1.7.1" +panel = "==1.3.6" +pip = "==23.3.1" +psutil = "==5.9.5" +python-dotenv = "==1.0.1" +ruff = "==0.6.0" +seaborn = "==0.13.2" + +[pypi-dependencies] +delu = "==0.0.25" +rtdl_num_embeddings = "==0.0.11" +rtdl_revisiting_models = "==0.0.2" + +[pypi-options] +index-url = "https://pypi.org/simple" + +[feature.cuda] +system-requirements = { libc = { family = "glibc", version = "2.27" } } +# In fact, it should be this: +# system-requirements = { cuda = "11.8", libc = { family = "glibc", version = "2.27" } } +# But the "cuda" requirement breaks the project on macOS. + +# (As of pixi==0.29.0) The ".target.linux-64" part is a workaround for projects +# with multiple platforms AND platform-specific dependencies (e.g. cuda) AND PyPI dependencies. +# Details: https://github.com/prefix-dev/pixi/issues/1051#issuecomment-2088820974 +[feature.cuda.target.linux-64.dependencies] +pytorch-cuda = { version = "==11.8", channel = "pytorch" } + +[environments] +default = { solve-group = "default" } +cuda = { features = ["cuda"], solve-group = "default" } diff --git a/tabm_integrated/paper/pyproject.toml b/tabm_integrated/paper/pyproject.toml new file mode 100644 index 000000000..151c00d4d --- /dev/null +++ b/tabm_integrated/paper/pyproject.toml @@ -0,0 +1,13 @@ +[tool.mypy] +check_untyped_defs = true +ignore_missing_imports = true + +[tool.ruff] +target-version = "py310" +line-length = 88 + +[tool.ruff.lint] +extend-select = ["RUF", "UP", "E501", "E731"] + +[tool.ruff.format] +quote-style = "single" diff --git a/tabm_integrated/paper/tools/cf_fisd_teachers.py b/tabm_integrated/paper/tools/cf_fisd_teachers.py new file mode 100644 index 000000000..19480f8c7 --- /dev/null +++ b/tabm_integrated/paper/tools/cf_fisd_teachers.py @@ -0,0 +1,310 @@ +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +if __name__ == '__main__': + _cwd = Path.cwd() + assert _cwd.joinpath( + 'pixi.toml' + ).exists(), 'The script must be run from the `paper/` directory' + sys.path.append(str(_cwd)) + del _cwd + +import delu +import numpy as np +import pandas as pd +try: + import tomllib +except ImportError: + import tomli as tomllib # type: ignore[no-redef] +from loguru import logger + +import lib +import lib.data + + +SUPPORTED_DATASETS = ( + 'sberbank-housing', + 'ecom-offers', + 'homesite-insurance', + 'cooking-time', + 'delivery-eta', +) +TEACHERS = ('xgb', 'lgbm', 'cat') + + +def _load_toml(path: Path) -> dict[str, Any]: + with path.open('rb') as f: + return tomllib.load(f) + + +def _build_combined_features(dataset) -> tuple[dict[str, pd.DataFrame], list[int]]: + parts = list(dataset.parts()) + n_num = dataset.n_num_features + n_bin = dataset.n_bin_features + n_cat = dataset.n_cat_features + + frames: dict[str, pd.DataFrame] = {part: pd.DataFrame() for part in parts} + for part in parts: + if 'x_num' in dataset: + frames[part] = pd.concat( + [frames[part], pd.DataFrame(dataset['x_num'][part])], axis=1 + ) + if 'x_cat' in dataset: + frames[part] = pd.concat( + [frames[part], pd.DataFrame(dataset['x_cat'][part].astype('int64'))], + axis=1, + ) + if 'x_bin' in dataset: + frames[part] = pd.concat( + [frames[part], pd.DataFrame(dataset['x_bin'][part].astype('int64'))], + axis=1, + ) + frames[part].columns = list(range(frames[part].shape[1])) + + cat_indices_in_combined: list[int] = [] + if 'x_cat' in dataset: + cat_indices_in_combined.extend(range(n_num, n_num + n_cat)) + if 'x_bin' in dataset: + cat_indices_in_combined.extend(range(n_num + n_cat, n_num + n_cat + n_bin)) + return frames, cat_indices_in_combined + + +def _train_xgb( + cfg: dict[str, Any], + X: dict[str, pd.DataFrame], + y: dict[str, np.ndarray], + cat_idx: list[int], + is_regression: bool, + is_binclass: bool, + seed: int, +) -> np.ndarray: + from xgboost import XGBClassifier, XGBRegressor + + df_X = {part: frame.astype(np.float32) for part, frame in X.items()} + + model_kwargs = dict(cfg['model']) + _force_cpu_in_model_cfg('xgb', model_kwargs) + extra = {'random_state': seed} + if is_regression: + model = XGBRegressor(**model_kwargs, **extra) + else: + eval_metric = 'auc' if is_binclass else 'merror' + model = XGBClassifier( + **model_kwargs, + **extra, + disable_default_eval_metric=True, + eval_metric=eval_metric, + ) + fit_kwargs = dict(cfg.get('fit', {})) + model.fit( + df_X['train'], + y['train'], + eval_set=[(df_X['val'], y['val'])], + **fit_kwargs, + ) + return np.asarray(model.feature_importances_, dtype=np.float64) + + +def _train_lgbm( + cfg: dict[str, Any], + X: dict[str, pd.DataFrame], + y: dict[str, np.ndarray], + cat_idx: list[int], + is_regression: bool, + is_binclass: bool, + seed: int, +) -> np.ndarray: + import lightgbm + from lightgbm import LGBMClassifier, LGBMRegressor + + model_kwargs = dict(cfg['model']) + _force_cpu_in_model_cfg('lgbm', model_kwargs) + stopping_rounds = model_kwargs.pop('stopping_rounds', None) + extra = {'random_state': seed} + if is_regression: + model = LGBMRegressor(**model_kwargs, **extra) + fit_extra = {'eval_metric': 'rmse'} + elif is_binclass: + model = LGBMClassifier(**model_kwargs, **extra) + fit_extra = {'eval_metric': 'auc'} + else: + model = LGBMClassifier(**model_kwargs, **extra) + fit_extra = {'eval_metric': 'multi_error'} + callbacks = [] + if stopping_rounds is not None: + callbacks.append(lightgbm.early_stopping(stopping_rounds=int(stopping_rounds))) + fit_kwargs = dict(cfg.get('fit', {})) + if cat_idx: + fit_extra['categorical_feature'] = list(cat_idx) + model.fit( + X['train'], + y['train'], + eval_set=[(X['val'], y['val'])], + callbacks=callbacks, + **fit_kwargs, + **fit_extra, + ) + importances = np.asarray(model.feature_importances_, dtype=np.float64) + if importances.sum() > 0.0: + return importances + return np.asarray( + model.booster_.feature_importance(importance_type='gain'), dtype=np.float64 + ) + + +def _train_cat( + cfg: dict[str, Any], + X: dict[str, pd.DataFrame], + y: dict[str, np.ndarray], + cat_idx: list[int], + is_regression: bool, + is_binclass: bool, + seed: int, +) -> np.ndarray: + from catboost import CatBoostClassifier, CatBoostRegressor, Pool + + model_kwargs = dict(cfg['model']) + _force_cpu_in_model_cfg('cat', model_kwargs) + model_kwargs.setdefault('random_seed', seed) + model_kwargs.setdefault('verbose', False) + model_kwargs.setdefault('allow_writing_files', False) + if is_regression: + model = CatBoostRegressor(**model_kwargs) + else: + model = CatBoostClassifier(**model_kwargs) + train_pool = Pool(X['train'], y['train'], cat_features=cat_idx if cat_idx else None) + val_pool = Pool(X['val'], y['val'], cat_features=cat_idx if cat_idx else None) + fit_kwargs = dict(cfg.get('fit', {})) + model.fit(train_pool, eval_set=val_pool, **fit_kwargs) + return np.asarray(model.get_feature_importance(type='PredictionValuesChange'), dtype=np.float64) + + +def _normalize_to_simplex(v: np.ndarray) -> np.ndarray: + v = np.clip(v.astype(np.float64), 0.0, None) + s = float(v.sum()) + if s > 0.0: + return (v / s).astype(np.float32) + return np.full_like(v, 1.0 / len(v), dtype=np.float32) + + +def _resolve_tuned_toml(tabred_root: Path, teacher: str, dataset: str) -> Path: + name_map = {'xgb': 'xgboost_', 'lgbm': 'lightgbm_', 'cat': 'catboost_'} + base = tabred_root / 'exp' / name_map[teacher] / dataset + candidates = [base / 'evaluation' / '0.toml', base / '0.toml'] + for p in candidates: + if p.exists(): + return p + return candidates[0] + + +def _force_cpu_in_model_cfg(teacher: str, model_cfg: dict[str, Any]) -> None: + if teacher == 'xgb': + model_cfg['device'] = 'cpu' + model_cfg['tree_method'] = model_cfg.get('tree_method', 'hist') + model_cfg.pop('gpu_id', None) + elif teacher == 'lgbm': + model_cfg['device_type'] = 'cpu' + model_cfg.pop('gpu_use_dp', None) + model_cfg.pop('gpu_platform_id', None) + model_cfg.pop('gpu_device_id', None) + elif teacher == 'cat': + model_cfg['task_type'] = 'CPU' + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument('--dataset', required=True, choices=SUPPORTED_DATASETS) + ap.add_argument('--seed', type=int, default=0) + ap.add_argument( + '--tabred-root', + type=Path, + default=Path(os.environ.get('TABRED_REPO_ROOT', 'D:/TabM_PROJ/tabred_fork')), + ) + ap.add_argument( + '--out-root', + type=Path, + default=Path('exp/cf_fisd/_teachers/tabred'), + ) + ap.add_argument('--teachers', nargs='+', default=list(TEACHERS), choices=list(TEACHERS)) + ap.add_argument('--data-path', default=None, + help='Override dataset path; defaults to data/.') + args = ap.parse_args() + + delu.random.seed(args.seed) + data_path = args.data_path or f'data/{args.dataset}' + dataset = lib.data.build_dataset(path=data_path, cache=True, cat_policy='ordinal') + if dataset.task.is_regression: + dataset.data['y'], _ = lib.data.standardize_labels(dataset.data['y']) + + X, cat_idx = _build_combined_features(dataset) + y = {part: np.asarray(dataset['y'][part]) for part in dataset.parts()} + + is_reg = dataset.task.is_regression + is_bin = bool(getattr(dataset.task, 'is_binclass', False)) + + out_dir = args.out_root / args.dataset + out_dir.mkdir(parents=True, exist_ok=True) + + n_num = dataset.n_num_features + n_cat = dataset.n_cat_features + n_bin = dataset.n_bin_features + n_features = n_num + n_cat + n_bin + logger.info( + f'{args.dataset}: n_num={n_num} n_cat={n_cat} n_bin={n_bin} n_features={n_features}' + ) + + trainers = {'xgb': _train_xgb, 'lgbm': _train_lgbm, 'cat': _train_cat} + for t in args.teachers: + toml_path = _resolve_tuned_toml(args.tabred_root, t, args.dataset) + if not toml_path.exists(): + raise FileNotFoundError(toml_path) + cfg = _load_toml(toml_path) + logger.info(f'training {t} for {args.dataset}: hp from {toml_path}') + importances = trainers[t]( + cfg=cfg, + X=X, + y=y, + cat_idx=cat_idx, + is_regression=is_reg, + is_binclass=is_bin, + seed=args.seed, + ) + if importances.shape != (n_features,): + raise ValueError( + f'importance shape {importances.shape} != ({n_features},) for {t}/{args.dataset}' + ) + importances = _normalize_to_simplex(importances) + out_path = out_dir / f'{t}.npy' + np.save(out_path, importances) + topk = np.argsort(importances)[::-1][:5].tolist() + entropy = float(-(importances * np.log(importances + 1e-12)).sum()) + logger.info( + f' saved {out_path} top5_idx={topk} entropy={entropy:.3f}' + ) + + meta = { + 'dataset': args.dataset, + 'task_type': dataset.task.type_.value, + 'n_num': n_num, + 'n_cat': n_cat, + 'n_bin': n_bin, + 'n_features': n_features, + 'cat_indices_in_combined': cat_idx, + 'feature_order': 'num,cat,bin', + 'seed': args.seed, + 'tabred_repo_root': str(args.tabred_root), + 'teachers': list(args.teachers), + } + (out_dir / 'meta.json').write_text(json.dumps(meta, indent=2)) + logger.info(f'wrote {out_dir / "meta.json"}') + + +if __name__ == '__main__': + lib.configure_libraries() + main() diff --git a/tabm_integrated/tools/aggregate_integrated_results.py b/tabm_integrated/tools/aggregate_integrated_results.py index 9d87935b7..bfde966e3 100644 --- a/tabm_integrated/tools/aggregate_integrated_results.py +++ b/tabm_integrated/tools/aggregate_integrated_results.py @@ -1,45 +1,327 @@ + from __future__ import annotations -import csv,json,math + +import argparse +import csv +import json +import math +import statistics +import tomllib +from collections import defaultdict from pathlib import Path -from statistics import mean,stdev -DATASETS=['sberbank-housing','ecom-offers','homesite-insurance','cooking-time','delivery-eta'] -VARIANTS=['baseline_plr','rla_only','esam_only','mfb_only','cf_fisd_only','all_four_combined','all_minus_rla','all_minus_esam','all_minus_mfb','all_minus_cf_fisd'] -PRIMARY={'sberbank-housing':'rmse','cooking-time':'rmse','delivery-eta':'rmse','ecom-offers':'roc-auc','homesite-insurance':'roc-auc'} -LOWER={'rmse'} -ROOT=Path(__file__).resolve().parents[1]; EXP=ROOT/'paper'/'exp'/'integrated' -def load_report(dataset,variant,seed): - path=EXP/dataset/f'{variant}-evaluation'/str(seed)/'report.json'; done=path.with_name('DONE') - if not path.exists() or not done.exists(): return None,path - r=json.loads(path.read_text()); return r,path -def delt(metric,v,b): return b-v if metric in LOWER else v-b -def fmt(x): return '' if x is None or (isinstance(x,float) and math.isnan(x)) else x -def main(): - summary=[]; audit=[] - for dataset in DATASETS: - metric=PRIMARY[dataset]; values={} - for variant in VARIANTS: - vals=[] - for seed in [0,1,2]: - r,path=load_report(dataset,variant,seed); ok=bool(r) and not r.get('failure') - audit.append({'dataset':dataset,'variant':variant,'seed':seed,'result_path':str(path.parent),'report_json_present':path.exists(),'DONE_present':(path.parent/'DONE').exists(),'failure_block_absent':ok,'gpu_name':(r or {}).get('gpu_name'),'amp_dtype':(r or {}).get('amp_dtype'),'inference_mode':(r or {}).get('inference_mode','mean'),'metric':metric,'test_metric':((r or {}).get('metrics',{}).get('test',{}).get(metric)),'failure':(r or {}).get('failure')}) - if ok: vals.append(float(r['metrics']['test'][metric])) - values[variant]=vals - base=values['baseline_plr'] - for variant in VARIANTS: - vals=values[variant] - if len(vals)==3 and len(base)==3: - m=mean(vals); s=stdev(vals); bm=mean(base); d=delt(metric,m,bm); pct=100*d/abs(bm); claim='baseline' if variant=='baseline_plr' else ('win' if d>0 else 'loss' if d<0 else 'tie') - elif vals: - m=mean(vals); s=stdev(vals) if len(vals)>1 else 0.0; bm=mean(base) if base else float('nan'); d=float('nan'); pct=float('nan'); claim='incomplete' - else: - m=s=bm=d=pct=float('nan'); claim='missing' - summary.append({'dataset':dataset,'variant':variant,'metric':metric,'metric_direction':'lower' if metric in LOWER else 'higher','inference_mode':'mean','precision':'bfloat16','n_seeds':len(vals),'mean':m,'std':s,'baseline_mean':bm,'absolute_delta':d,'percent_delta':pct,'safe_claim_status':claim,'config_path':str(EXP/dataset/f'{variant}-evaluation/0.toml'),'result_path':str(EXP/dataset/f'{variant}-evaluation')}) - for name,rows in [('final_integrated_summary.csv',summary),('final_integrated_audit.csv',audit)]: - path=ROOT/'paper'/'exp'/name; path.parent.mkdir(parents=True,exist_ok=True) - with path.open('w',newline='') as f: w=csv.DictWriter(f,fieldnames=list(rows[0].keys()),lineterminator='\n'); w.writeheader(); w.writerows(rows) - lines=['# Final Integrated TabM Experiment Report','','Mean ± std over available seeds. Safe claims require all 3 seeds.','','| dataset | variant | metric | direction | inference | precision | mean ± std | delta | percent delta | n | status | config path | result path |','|---|---|---|---|---|---|---:|---:|---:|---:|---|---|---|'] - for r in summary: - lines.append(f"| {r['dataset']} | {r['variant']} | {r['metric']} | {r['metric_direction']} | {r['inference_mode']} | {r['precision']} | {float(r['mean']):.6g} ± {float(r['std']):.3g} | {float(r['absolute_delta']):.6g} | {float(r['percent_delta']):.3f}% | {r['n_seeds']} | {r['safe_claim_status']} | {r['config_path']} | {r['result_path']} |") - (ROOT/'FINAL_EXPERIMENT_REPORT.md').write_text('\n'.join(lines)+'\n') - print(ROOT/'paper'/'exp'/'final_integrated_summary.csv'); print(ROOT/'FINAL_EXPERIMENT_REPORT.md') -if __name__=='__main__': main() +from typing import Any + +import tomli_w + +PAPER = Path(__file__).resolve().parents[1] / 'paper' +EXP_ROOT = PAPER / 'exp' / 'integrated' +SUMMARY_PATH = PAPER / 'exp' / 'final_integrated_summary.csv' +AUDIT_PATH = PAPER / 'exp' / 'final_integrated_audit.csv' +REPORT_PATH = PAPER.parent / 'FINAL_EXPERIMENT_REPORT.md' +SELECTED_PATH = PAPER / 'exp' / 'selected_integrated_configs.csv' +FINAL_MANIFEST = EXP_ROOT / 'manifest_final.txt' + +DATASET_ORDER = ['sberbank-housing', 'ecom-offers', 'homesite-insurance', 'cooking-time', 'delivery-eta'] +FINAL_DISPLAY = ['baseline_plr', 'best_rla_only', 'best_esam_only', 'best_mfb_only', 'best_cf_fisd_only', 'best_combined'] + + +def infer_inference_mode(path: Path) -> str: + parent = path.parent.parent.name + if parent.endswith('-best-head-evaluation'): + return 'best-head' + if parent.endswith('-greedy-heads-evaluation'): + return 'greedy-heads' + return 'mean' + + +def mean_config_dir_for_result_dir(run_dir: Path) -> Path: + eval_dir = run_dir.parent + name = eval_dir.name + if name.endswith('-best-head-evaluation'): + mean_name = name.removesuffix('-best-head-evaluation') + '-evaluation' + elif name.endswith('-greedy-heads-evaluation'): + mean_name = name.removesuffix('-greedy-heads-evaluation') + '-evaluation' + else: + mean_name = name + return eval_dir.with_name(mean_name) / run_dir.name + + +def variant_from_result(path: Path) -> str: + name = path.parent.parent.name + for suffix in ['-best-head-evaluation', '-greedy-heads-evaluation', '-evaluation']: + if name.endswith(suffix): + return name.removesuffix(suffix) + return name + + +def dataset_from_result(path: Path) -> str: + return path.parent.parent.parent.name + + +def wave_from_result(path: Path) -> str: + return path.parent.parent.parent.parent.name + + +def score_key(report: dict[str, Any]) -> tuple[str, str, str]: + val = report.get('metrics', {}).get('val', {}) + if 'rmse' in val: + return 'RMSE', 'rmse', 'lower' + if 'roc-auc' in val: + return 'AUROC', 'roc-auc', 'higher' + if 'accuracy' in val: + return 'accuracy', 'accuracy', 'higher' + raise KeyError(f'Unknown metrics keys: {sorted(val)}') + + +def signed_delta(test_mean: float, baseline_mean: float, direction: str) -> tuple[float, float]: + if direction == 'lower': + delta = baseline_mean - test_mean + pct = 100.0 * delta / baseline_mean if baseline_mean else float('nan') + else: + delta = test_mean - baseline_mean + pct = 100.0 * delta / abs(baseline_mean) if baseline_mean else float('nan') + return delta, pct + + +def status_for(delta: float, baseline_std: float, n: int, invalid: bool) -> str: + if invalid or n != 3: + return 'invalid' + if abs(delta) <= 1e-12: + return 'tie' + if delta < 0: + return 'loss' + if abs(delta) > baseline_std: + return 'clear_win' + return 'weak_win' + + +def iter_reports(root: Path = EXP_ROOT): + for report_path in root.rglob('report.json'): + if not report_path.parent.joinpath('DONE').exists(): + continue + try: + report = json.loads(report_path.read_text()) + except Exception: + continue + mean_run_dir = mean_config_dir_for_result_dir(report_path.parent) + cfg_path = mean_run_dir.with_suffix('.toml') + yield report_path, cfg_path, report + + +def collect_rows() -> list[dict[str, Any]]: + rows = [] + for report_path, cfg_path, report in iter_reports(): + dataset = dataset_from_result(report_path) + variant = variant_from_result(report_path) + wave = wave_from_result(report_path) + inference = infer_inference_mode(report_path) + metric, key, direction = score_key(report) + rows.append({ + 'wave': wave, + 'dataset': dataset, + 'variant': variant, + 'seed': int(report.get('config', {}).get('seed', report_path.parent.name)), + 'metric': metric, + 'metric_key': key, + 'direction': direction, + 'validation_metric': float(report['metrics']['val'][key]), + 'validation_score': float(report['metrics']['val']['score']), + 'test_metric': float(report['metrics']['test'][key]), + 'inference_mode': inference, + 'config_path': str(cfg_path.relative_to(PAPER)) if cfg_path.exists() else str(cfg_path), + 'result_path': str(report_path.parent.relative_to(PAPER)), + 'failure': bool(report.get('failure')), + 'best_step': report.get('best_step'), + }) + return rows + + +def candidate_family(variant: str) -> str | None: + if variant == 'baseline_plr': + return 'baseline_plr' + if variant == 'rla_only' or variant.startswith('rla_rank'): + return 'best_rla_only' + if variant == 'esam_only' or variant.startswith('esam_rho'): + return 'best_esam_only' + if variant == 'mfb_only' or variant.startswith('mfb_keep'): + return 'best_mfb_only' + if variant == 'cf_fisd_only' or variant.startswith('cf_fisd_lambda'): + return 'best_cf_fisd_only' + if variant in {'all_four_combined','rla_esam','rla_mfb','rla_cf_fisd','esam_mfb','esam_cf_fisd','mfb_cf_fisd','all_minus_rla','all_minus_esam','all_minus_mfb','all_minus_cf_fisd'}: + return 'best_combined' + return None + + +def select_final_configs() -> None: + rows = [r for r in collect_rows() if r['wave'] in {'smoke', 'sweeps'} and r['seed'] == 0 and not r['failure']] + by = defaultdict(list) + for row in rows: + fam = candidate_family(row['variant']) + if fam and fam != 'baseline_plr': + by[(row['dataset'], fam)].append(row) + selected = [] + for dataset in DATASET_ORDER: + baseline_cfg = EXP_ROOT / 'smoke' / dataset / 'baseline_plr-evaluation' / '0.toml' + if not baseline_cfg.exists(): + baseline_cfg = EXP_ROOT / 'baseline_fidelity' / dataset / 'baseline_plr-evaluation' / '0.toml' + selected.append({'dataset': dataset, 'final_variant': 'baseline_plr', 'source_variant': 'baseline_plr', 'source_wave': 'smoke', 'inference_mode': 'mean', 'validation_metric': '', 'validation_score': '', 'source_config_path': str(baseline_cfg.relative_to(PAPER))}) + for fam in FINAL_DISPLAY[1:]: + candidates = by.get((dataset, fam), []) + if not candidates: + raise RuntimeError(f'No selection candidates for {dataset}/{fam}') + best = max(candidates, key=lambda r: r['validation_score']) + selected.append({'dataset': dataset, 'final_variant': fam, 'source_variant': best['variant'], 'source_wave': best['wave'], 'inference_mode': best['inference_mode'], 'validation_metric': best['validation_metric'], 'validation_score': best['validation_score'], 'source_config_path': best['config_path']}) + + SELECTED_PATH.parent.mkdir(parents=True, exist_ok=True) + with SELECTED_PATH.open('w', newline='') as f: + writer = csv.DictWriter(f, fieldnames=list(selected[0].keys()), lineterminator="\n") + writer.writeheader() + writer.writerows(selected) + + final_paths: list[Path] = [] + for row in selected: + source_cfg = PAPER / row['source_config_path'] + cfg = tomllib.loads(source_cfg.read_text()) + for seed in [0, 1, 2]: + cfg_seed = json.loads(json.dumps(cfg)) + cfg_seed['seed'] = seed + if cfg_seed.get('model', {}).get('mfb'): + cfg_seed['model']['mfb']['mask_seed'] = seed + out = EXP_ROOT / 'final' / row['dataset'] / f"{row['final_variant']}-evaluation" / f'{seed}.toml' + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(tomli_w.dumps(cfg_seed)) + final_paths.append(out) + FINAL_MANIFEST.write_text('\n'.join(str(p.relative_to(PAPER)) for p in final_paths) + '\n') + print(f'selected configs: {SELECTED_PATH.relative_to(PAPER)}') + print(f'final manifest: {FINAL_MANIFEST.relative_to(PAPER)} ({len(final_paths)} jobs)') + + +def summarize_final() -> list[dict[str, Any]]: + selection = {} + if SELECTED_PATH.exists(): + with SELECTED_PATH.open() as f: + for row in csv.DictReader(f): + selection[(row['dataset'], row['final_variant'])] = row + rows = [r for r in collect_rows() if r['wave'] == 'final'] + wanted = set((d, v) for d in DATASET_ORDER for v in FINAL_DISPLAY) + grouped = defaultdict(list) + for row in rows: + if (row['dataset'], row['variant']) not in wanted: + continue + sel = selection.get((row['dataset'], row['variant'])) + if sel and row['inference_mode'] != sel['inference_mode']: + continue + if row['variant'] == 'baseline_plr' and row['inference_mode'] != 'mean': + continue + grouped[(row['dataset'], row['variant'])].append(row) + + out = [] + baseline_stats = {} + for dataset in DATASET_ORDER: + vals = [r['test_metric'] for r in grouped.get((dataset, 'baseline_plr'), [])] + baseline_stats[dataset] = (statistics.mean(vals), statistics.stdev(vals) if len(vals) > 1 else 0.0) if vals else (float('nan'), float('nan')) + + for dataset in DATASET_ORDER: + for variant in FINAL_DISPLAY: + rs = sorted(grouped.get((dataset, variant), []), key=lambda r: r['seed']) + n = len(rs) + invalid = n != 3 or any(r['failure'] for r in rs) + metric = rs[0]['metric'] if rs else '' + direction = rs[0]['direction'] if rs else '' + val_mean = statistics.mean([r['validation_metric'] for r in rs]) if rs else float('nan') + test_values = [r['test_metric'] for r in rs] + test_mean = statistics.mean(test_values) if test_values else float('nan') + test_std = statistics.stdev(test_values) if len(test_values) > 1 else 0.0 + base_mean, base_std = baseline_stats[dataset] + delta, pct = signed_delta(test_mean, base_mean, direction) if rs else (float('nan'), float('nan')) + status = 'baseline' if variant == 'baseline_plr' else status_for(delta, base_std, n, invalid) + sel = selection.get((dataset, variant), {}) + out.append({ + 'dataset': dataset, + 'variant': variant, + 'source_variant': sel.get('source_variant', variant), + 'metric': metric, + 'direction': direction, + 'validation_metric': val_mean, + 'test_metric': test_mean, + 'mean': test_mean, + 'std': test_std, + 'n_seeds': n, + 'baseline_mean': base_mean, + 'absolute_delta': delta, + 'percent_delta': pct, + 'precision': '3 seeds', + 'inference_mode': sel.get('inference_mode', 'mean' if variant == 'baseline_plr' else ''), + 'config_path': ';'.join(r['config_path'] for r in rs), + 'result_path': ';'.join(r['result_path'] for r in rs), + 'status': status, + }) + return out + + +def write_csv(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open('w', newline='') as f: + writer = csv.DictWriter(f, fieldnames=list(rows[0].keys()) if rows else [], lineterminator="\n") + writer.writeheader() + writer.writerows(rows) + + +def fmt(x: Any) -> str: + if isinstance(x, float): + if math.isnan(x): + return 'nan' + return f'{x:.6g}' + return str(x) + + +def write_report(summary_rows: list[dict[str, Any]]) -> None: + lines = ['# Integrated TabM final experiment report', ''] + lines += ['Official TabM PLR/PiecewiseLinearEmbeddings baseline configs are preserved exactly; variants add only module flags for RLA, ESAM, MFB, and CF-FISD.', ''] + lines += ['## Final 3-seed results', ''] + lines += ['| dataset | variant | source config | metric | inference | mean ± std | delta | status |', '|---|---|---|---|---|---:|---:|---|'] + for r in summary_rows: + mean_std = f"{fmt(r['mean'])} ± {fmt(r['std'])}" + lines.append(f"| {r['dataset']} | {r['variant']} | {r['source_variant']} | {r['metric']} | {r['inference_mode']} | {mean_std} | {fmt(r['absolute_delta'])} | {r['status']} |") + lines += ['', '## Validation-selected configs', ''] + if SELECTED_PATH.exists(): + lines += ['| dataset | final variant | selected sweep variant | inference | validation metric |', '|---|---|---|---|---:|'] + with SELECTED_PATH.open() as f: + for row in csv.DictReader(f): + lines.append(f"| {row['dataset']} | {row['final_variant']} | {row['source_variant']} | {row['inference_mode']} | {row['validation_metric']} |") + lines += ['', '## Module wins vs baseline', ''] + for variant in FINAL_DISPLAY[1:]: + wins = [r['dataset'] for r in summary_rows if r['variant'] == variant and r['status'] in {'clear_win','weak_win'}] + lines.append(f"- `{variant}`: {', '.join(wins) if wins else 'none'}") + REPORT_PATH.write_text('\n'.join(lines) + '\n') + + +def stage_wave(manifest: str) -> None: + rows = collect_rows() + print(f'aggregated complete reports: {len(rows)}') + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--stage', choices=['wave', 'select-final', 'final'], default='final') + parser.add_argument('--manifest') + args = parser.parse_args() + if args.stage == 'wave': + stage_wave(args.manifest or '') + return + if args.stage == 'select-final': + select_final_configs() + return + summary = summarize_final() + write_csv(SUMMARY_PATH, summary) + audit_rows = collect_rows() + if audit_rows: + write_csv(AUDIT_PATH, audit_rows) + write_report(summary) + print(f'summary: {SUMMARY_PATH.relative_to(PAPER)}') + print(f'audit: {AUDIT_PATH.relative_to(PAPER)}') + print(f'report: {REPORT_PATH.relative_to(PAPER.parent)}') + + +if __name__ == '__main__': + main() diff --git a/tabm_integrated/tools/generate_integrated_configs.py b/tabm_integrated/tools/generate_integrated_configs.py index e5ec0e0d2..396b06fc7 100644 --- a/tabm_integrated/tools/generate_integrated_configs.py +++ b/tabm_integrated/tools/generate_integrated_configs.py @@ -1,27 +1,294 @@ + from __future__ import annotations + +import argparse +import copy +import shutil from pathlib import Path +from typing import Any + import tomli_w -DATASETS=['sberbank-housing','ecom-offers','homesite-insurance','cooking-time','delivery-eta'] -VARIANTS={ - 'baseline_plr':{}, - 'rla_only':{'rla_enabled':True,'rla_rank':4,'rla_scale':0.05}, - 'esam_only':{'esam_enabled':True,'esam_rho':0.0025}, - 'mfb_only':{'mfb_enabled':True,'mfb_keep':0.8}, - 'cf_fisd_only':{'cf_fisd_enabled':True}, - 'all_four_combined':{'rla_enabled':True,'rla_rank':4,'rla_scale':0.05,'esam_enabled':True,'esam_rho':0.0025,'mfb_enabled':True,'mfb_keep':0.8,'cf_fisd_enabled':True}, - 'all_minus_rla':{'esam_enabled':True,'esam_rho':0.0025,'mfb_enabled':True,'mfb_keep':0.8,'cf_fisd_enabled':True}, - 'all_minus_esam':{'rla_enabled':True,'rla_rank':4,'rla_scale':0.05,'mfb_enabled':True,'mfb_keep':0.8,'cf_fisd_enabled':True}, - 'all_minus_mfb':{'rla_enabled':True,'rla_rank':4,'rla_scale':0.05,'esam_enabled':True,'esam_rho':0.0025,'cf_fisd_enabled':True}, - 'all_minus_cf_fisd':{'rla_enabled':True,'rla_rank':4,'rla_scale':0.05,'esam_enabled':True,'esam_rho':0.0025,'mfb_enabled':True,'mfb_keep':0.8}, + +PAPER = Path(__file__).resolve().parents[1] / 'paper' +EXP_ROOT = PAPER / 'exp' / 'integrated' +TEACHER_ROOT = Path('../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred') + +DATASET_ORDER = [ + 'sberbank-housing', + 'ecom-offers', + 'homesite-insurance', + 'cooking-time', + 'delivery-eta', +] + +OFFICIAL_BASE: dict[str, dict[str, Any]] = { + 'sberbank-housing': { + 'batch_size': 256, + 'patience': 16, + 'n_epochs': -1, + 'gradient_clipping_norm': 1.0, + 'amp': True, + 'data': {'cache': True, 'path': 'data/sberbank-housing', 'num_policy': 'noisy-quantile', 'cat_policy': 'ordinal'}, + 'optimizer': {'lr': 0.0008922700423431547, 'weight_decay': 0.00021333759467820313}, + 'model': { + 'arch_type': 'tabm', 'k': 32, 'share_training_batches': False, + 'backbone': {'n_blocks': 3, 'd_block': 256, 'dropout': 0.3128700072370906}, + 'num_embeddings': {'type': 'PiecewiseLinearEmbeddings', 'd_embedding': 28}, + }, + 'bins': {'n_bins': 84}, + }, + 'ecom-offers': { + 'batch_size': 1024, + 'patience': 16, + 'n_epochs': -1, + 'gradient_clipping_norm': 1.0, + 'amp': True, + 'data': {'cache': True, 'path': 'data/ecom-offers', 'num_policy': 'noisy-quantile'}, + 'optimizer': {'lr': 0.00024262819114537424, 'weight_decay': 0.0001501852317298042}, + 'model': { + 'arch_type': 'tabm', 'k': 32, 'share_training_batches': False, + 'backbone': {'n_blocks': 1, 'd_block': 960, 'dropout': 0.0}, + 'num_embeddings': {'type': 'PiecewiseLinearEmbeddings', 'd_embedding': 16}, + }, + 'bins': {'n_bins': 47}, + }, + 'homesite-insurance': { + 'batch_size': 1024, + 'patience': 16, + 'n_epochs': -1, + 'gradient_clipping_norm': 1.0, + 'amp': True, + 'data': {'cache': True, 'path': 'data/homesite-insurance', 'num_policy': 'noisy-quantile', 'cat_policy': 'ordinal'}, + 'optimizer': {'lr': 0.0018580623030886075, 'weight_decay': 0.0001614529849348179}, + 'model': { + 'arch_type': 'tabm', 'k': 32, 'share_training_batches': False, + 'backbone': {'n_blocks': 3, 'd_block': 704, 'dropout': 0.0}, + 'num_embeddings': {'type': 'PiecewiseLinearEmbeddings', 'd_embedding': 8}, + }, + 'bins': {'n_bins': 15}, + }, + 'cooking-time': { + 'batch_size': 1024, + 'patience': 16, + 'n_epochs': -1, + 'gradient_clipping_norm': 1.0, + 'amp': True, + 'data': {'cache': True, 'path': 'data/cooking-time', 'cat_policy': 'ordinal'}, + 'optimizer': {'lr': 0.00012065020494450812, 'weight_decay': 0.07000081679295954}, + 'model': { + 'arch_type': 'tabm', 'k': 32, 'share_training_batches': False, + 'backbone': {'n_blocks': 1, 'd_block': 416, 'dropout': 0.0}, + 'num_embeddings': {'type': 'PiecewiseLinearEmbeddings', 'd_embedding': 16}, + }, + 'bins': {'n_bins': 8}, + }, + 'delivery-eta': { + 'batch_size': 1024, + 'patience': 16, + 'n_epochs': -1, + 'gradient_clipping_norm': 1.0, + 'amp': True, + 'data': {'cache': True, 'path': 'data/delivery-eta', 'cat_policy': 'ordinal'}, + 'optimizer': {'lr': 0.0025639267063470926, 'weight_decay': 0.0507074044872309}, + 'model': { + 'arch_type': 'tabm', 'k': 32, 'share_training_batches': False, + 'backbone': {'n_blocks': 2, 'd_block': 752, 'dropout': 0.18437573041970334}, + 'num_embeddings': {'type': 'PiecewiseLinearEmbeddings', 'd_embedding': 32}, + }, + 'bins': {'n_bins': 81}, + }, } -BATCH={'sberbank-housing':1024,'ecom-offers':2048,'homesite-insurance':2048,'cooking-time':2048,'delivery-eta':2048} -def cfg(dataset,variant,seed): - return {'dataset':dataset,'variant':variant,'seed':seed,'batch_size':BATCH[dataset],'patience':6,'n_epochs':20,'gradient_clipping_norm':1.0,'amp':True,'data':{'path':f'data/{dataset}','num_policy':'standard','cache':False},'optimizer':{'type':'AdamW','lr':3e-4,'weight_decay':1e-5},'model':{'arch_type':'tabm','k':16,'n_blocks':2,'d_block':384,'dropout':0.1,'d_embedding':16,'n_bins':48,**VARIANTS[variant]}} -def main(): - root=Path(__file__).resolve().parents[1]/'paper'/'exp'/'integrated' - for seed in [0,1,2]: - for dataset in DATASETS: - for variant in VARIANTS: - d=root/dataset/f'{variant}-evaluation'; d.mkdir(parents=True,exist_ok=True); (d/f'{seed}.toml').write_text(tomli_w.dumps(cfg(dataset,variant,seed))) - print(root) -if __name__=='__main__': main() + +MODULE_DEFAULTS = { + 'rla': {'rank': 4, 'noise': 1e-4, 'inference': 'mean'}, + 'esam': {'rho': 0.005}, + 'mfb': {'keep': 0.90}, + 'cf_fisd': {'lambda': 0.10}, +} + + +def base_config(dataset: str, seed: int) -> dict[str, Any]: + cfg = copy.deepcopy(OFFICIAL_BASE[dataset]) + cfg['seed'] = int(seed) + cfg['head_selection'] = True + return cfg + + +def apply_rla(cfg: dict[str, Any], *, rank: int = 4, noise: float = 1e-4) -> None: + cfg['model']['rla_rank'] = int(rank) + cfg['model']['rla_first_only'] = False + cfg['model']['rla_additive'] = False + cfg['model']['rla_init'] = 'base_preserving' + cfg['model']['rla_base_preserve_noise'] = float(noise) + cfg['rla_adapter_lr_multiplier'] = 1.0 + cfg['rla_extra_paths_freeze_fraction'] = 0.0 + + +def apply_esam(cfg: dict[str, Any], *, rho: float = 0.005) -> None: + cfg['use_esam'] = True + cfg['esam_rho'] = float(rho) + cfg['esam_eps'] = 1e-12 + cfg['esam_adapter_only'] = True + cfg['esam_memberwise'] = True + cfg['esam_warmup_epochs'] = 0 + cfg['esam_start_epoch'] = 0 + cfg['esam_end_epoch'] = -1 + cfg['esam_log_diagnostics'] = False + cfg['esam_diagnostics_every'] = 100 + + +def apply_mfb(cfg: dict[str, Any], *, keep: float = 0.90) -> None: + cfg['model']['mfb'] = { + 'enabled': True, + 'mask_mode': 'member_fixed', + 'mask_granularity': 'feature_group', + 'keep_rate': float(keep), + 'training_only': False, + 'inverted_scaling': True, + 'use_soft_mask': False, + 'mask_strength': 1.0, + 'anchor_fraction': 0.0, + 'warmup_epochs': 0, + 'mask_seed': int(cfg['seed']), + } + + +def apply_cf_fisd(cfg: dict[str, Any], dataset: str, *, lam: float = 0.10) -> None: + cfg['cf_fisd'] = { + 'lambda': float(lam), + 'variant': 'raw', + 'dataset_name': dataset, + 'teacher_dir': str(TEACHER_ROOT / dataset), + 'teacher_names': ['xgb', 'lgbm', 'cat'], + } + + +def make_variant_config(dataset: str, seed: int, variant: str) -> dict[str, Any]: + cfg = base_config(dataset, seed) + if variant == 'baseline_plr': + return cfg + if variant.startswith('rla_rank'): + parts = variant.split('_') + apply_rla(cfg, rank=int(parts[1].removeprefix('rank')), noise=float(parts[2].removeprefix('noise'))) + return cfg + if variant.startswith('esam_rho'): + apply_esam(cfg, rho=float(variant.removeprefix('esam_rho'))) + return cfg + if variant.startswith('mfb_keep'): + apply_mfb(cfg, keep=float(variant.removeprefix('mfb_keep'))) + return cfg + if variant.startswith('cf_fisd_lambda'): + apply_cf_fisd(cfg, dataset, lam=float(variant.removeprefix('cf_fisd_lambda'))) + return cfg + + modules: set[str] + if variant == 'rla_only': + modules = {'rla'} + elif variant == 'esam_only': + modules = {'esam'} + elif variant == 'mfb_only': + modules = {'mfb'} + elif variant == 'cf_fisd_only': + modules = {'cf_fisd'} + elif variant == 'all_four_combined': + modules = {'rla', 'esam', 'mfb', 'cf_fisd'} + elif variant == 'rla_esam': + modules = {'rla', 'esam'} + elif variant == 'rla_mfb': + modules = {'rla', 'mfb'} + elif variant == 'rla_cf_fisd': + modules = {'rla', 'cf_fisd'} + elif variant == 'esam_mfb': + modules = {'esam', 'mfb'} + elif variant == 'esam_cf_fisd': + modules = {'esam', 'cf_fisd'} + elif variant == 'mfb_cf_fisd': + modules = {'mfb', 'cf_fisd'} + elif variant == 'all_minus_rla': + modules = {'esam', 'mfb', 'cf_fisd'} + elif variant == 'all_minus_esam': + modules = {'rla', 'mfb', 'cf_fisd'} + elif variant == 'all_minus_mfb': + modules = {'rla', 'esam', 'cf_fisd'} + elif variant == 'all_minus_cf_fisd': + modules = {'rla', 'esam', 'mfb'} + else: + raise KeyError(variant) + if 'rla' in modules: + apply_rla(cfg, rank=MODULE_DEFAULTS['rla']['rank'], noise=MODULE_DEFAULTS['rla']['noise']) + if 'esam' in modules: + apply_esam(cfg, rho=MODULE_DEFAULTS['esam']['rho']) + if 'mfb' in modules: + apply_mfb(cfg, keep=MODULE_DEFAULTS['mfb']['keep']) + if 'cf_fisd' in modules: + apply_cf_fisd(cfg, dataset, lam=MODULE_DEFAULTS['cf_fisd']['lambda']) + return cfg + + +def write_config(wave: str, dataset: str, variant: str, seed: int, *, force: bool) -> Path: + cfg = make_variant_config(dataset, seed, variant) + path = EXP_ROOT / wave / dataset / f'{variant}-evaluation' / f'{seed}.toml' + path.parent.mkdir(parents=True, exist_ok=True) + if force or not path.exists(): + path.write_text(tomli_w.dumps(cfg)) + return path + + +def write_manifest(name: str, paths: list[Path]) -> Path: + manifest = EXP_ROOT / f'manifest_{name}.txt' + manifest.parent.mkdir(parents=True, exist_ok=True) + manifest.write_text('\n'.join(str(p.relative_to(PAPER)) for p in paths) + ('\n' if paths else '')) + return manifest + + +def configs_for_stage(stage: str) -> dict[str, list[Path]]: + out: dict[str, list[Path]] = {} + if stage in {'initial', 'baseline_fidelity'}: + out['baseline_fidelity'] = [write_config('baseline_fidelity', d, 'baseline_plr', 0, force=True) for d in DATASET_ORDER] + if stage in {'initial', 'smoke'}: + variants = ['baseline_plr', 'rla_only', 'esam_only', 'mfb_only', 'cf_fisd_only', 'all_four_combined'] + out['smoke'] = [write_config('smoke', d, v, 0, force=True) for d in DATASET_ORDER for v in variants] + if stage in {'initial', 'sweeps'}: + variants: list[str] = [] + for rank in [2, 4, 8]: + for noise in [1e-3, 1e-4, 1e-5]: + variants.append(f'rla_rank{rank}_noise{noise:g}') + variants += [f'esam_rho{rho:g}' for rho in [0.001, 0.0025, 0.005, 0.01]] + variants += [f'mfb_keep{keep:g}' for keep in [0.70, 0.80, 0.90, 0.95]] + variants += [f'cf_fisd_lambda{lam:g}' for lam in [0.05, 0.10, 0.20]] + variants += ['rla_esam', 'rla_mfb', 'rla_cf_fisd', 'esam_mfb', 'esam_cf_fisd', 'mfb_cf_fisd', 'all_four_combined', 'all_minus_rla', 'all_minus_esam', 'all_minus_mfb', 'all_minus_cf_fisd'] + out['sweeps'] = [write_config('sweeps', d, v, 0, force=True) for d in DATASET_ORDER for v in variants] + return out + + +def verify_baseline_configs(paths: list[Path]) -> None: + import tomllib + for path in paths: + dataset = path.parts[-3] + cfg = tomllib.loads(path.read_text()) + expected = copy.deepcopy(OFFICIAL_BASE[dataset]) + for key, value in expected.items(): + if cfg.get(key) != value: + raise AssertionError(f'{path}: {key} differs from official template') + if cfg['seed'] != 0 or cfg.get('head_selection') is not True: + raise AssertionError(f'{path}: seed/head_selection wrapper invalid') + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--stage', choices=['initial', 'baseline_fidelity', 'smoke', 'sweeps'], default='initial') + parser.add_argument('--clean', action='store_true') + args = parser.parse_args() + if args.clean and EXP_ROOT.exists(): + shutil.rmtree(EXP_ROOT) + generated = configs_for_stage(args.stage) + for name, paths in generated.items(): + manifest = write_manifest(name, paths) + print(f'{name}: {len(paths)} configs -> {manifest.relative_to(PAPER)}') + if 'baseline_fidelity' in generated: + verify_baseline_configs(generated['baseline_fidelity']) + print('baseline_fidelity config check: OK') + + +if __name__ == '__main__': + main() diff --git a/tabm_integrated/tools/run_integrated_matrix.sh b/tabm_integrated/tools/run_integrated_matrix.sh index 8ef95f4f1..790f692a3 100755 --- a/tabm_integrated/tools/run_integrated_matrix.sh +++ b/tabm_integrated/tools/run_integrated_matrix.sh @@ -1,79 +1,112 @@ #!/usr/bin/env bash set -euo pipefail -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -PAPER="$ROOT/paper" + +PAPER_DIR="${PAPER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../paper" && pwd)}" +MANIFEST="${1:-${MANIFEST:-}}" +if [[ -z "$MANIFEST" ]]; then + echo "usage: $0 paper/exp/integrated/manifest_*.txt" >&2 + exit 2 +fi +cd "$PAPER_DIR" +if [[ ! -f "$MANIFEST" ]]; then + echo "manifest not found: $MANIFEST" >&2 + exit 2 +fi + PYTHON="${PYTHON:-/workspace/.venvs/tabm_integrated/bin/python}" N_GPUS="${N_GPUS:-16}" -SEEDS="${SEEDS:-0}" -VARIANTS="${VARIANTS:-baseline_plr rla_only esam_only mfb_only cf_fisd_only all_four_combined}" -DATASETS="sberbank-housing ecom-offers homesite-insurance cooking-time delivery-eta" -LOG_ROOT="$PAPER/exp/integrated/_logs" -QUEUE="$PAPER/exp/integrated/_queue_${SEEDS// /_}.txt" +FORCE="${FORCE:-0}" +LOG_ROOT="exp/integrated/_logs/$(basename "$MANIFEST" .txt)_$(date +%Y%m%d_%H%M%S)" mkdir -p "$LOG_ROOT" -"$PYTHON" "$ROOT/tools/generate_integrated_configs.py" -: > "$QUEUE" -for seed in $SEEDS; do - for dataset in $DATASETS; do - for variant in $VARIANTS; do - cfg="$PAPER/exp/integrated/$dataset/$variant-evaluation/$seed.toml" - out="$PAPER/exp/integrated/$dataset/$variant-evaluation/$seed" - if [[ -f "$out/DONE" && -f "$out/report.json" ]] && "$PYTHON" - "$out/report.json" <<'PYCHK' +QUEUE="exp/integrated/_queue_$(basename "$MANIFEST" .txt)_$$.txt" +LOCK="$QUEUE.lock" +cp "$MANIFEST" "$QUEUE" +: > "$LOCK" +USAGE_CSV="$LOG_ROOT/gpu_usage.csv" +echo "gpu,config,started_at" > "$USAGE_CSV" + +cleanup_artifacts() { + local out="$1" + rm -f "$out/checkpoint.pt" "$out/checkpoint_best.pt" "$out/predictions.npz" "$out/summary.json" + rm -f "$out"/events.out.tfevents.* 2>/dev/null || true +} + +check_done() { + local out="$1" + "$PYTHON" - "$out" <<'PY' import json, sys -r=json.load(open(sys.argv[1])) -sys.exit(1 if r.get('failure') else 0) -PYCHK - then - echo "reuse $dataset $variant $seed" - else - echo "$cfg|$out|$dataset|$variant|$seed" >> "$QUEUE" - fi - done - done -done +from pathlib import Path +out=Path(sys.argv[1]) +report=out/'report.json' +done=out/'DONE' +if not done.exists() or not report.exists(): + raise SystemExit(1) +payload=json.loads(report.read_text()) +if payload.get('failure'): + raise SystemExit(2) +PY +} + worker() { - gpu="$1" + local gpu="$1" + export CUDA_VISIBLE_DEVICES="$gpu" while true; do - line="" - exec 9<>"$QUEUE.lock" - flock 9 - if [[ -s "$QUEUE" ]]; then - line="$(head -n 1 "$QUEUE")" - tail -n +2 "$QUEUE" > "$QUEUE.tmp" - mv "$QUEUE.tmp" "$QUEUE" - fi - flock -u 9 - [[ -z "$line" ]] && break - IFS='|' read -r cfg out dataset variant seed <<< "$line" - mkdir -p "$(dirname "$out")" "$LOG_ROOT/$dataset/$variant" - log="$LOG_ROOT/$dataset/$variant/seed${seed}.gpu${gpu}.log" - echo "START $(date -Iseconds) gpu=$gpu dataset=$dataset variant=$variant seed=$seed" | tee "$log" - if ! (cd "$PAPER" && CUDA_VISIBLE_DEVICES="$gpu" "$PYTHON" "$PAPER/bin/run_integrated.py" "$cfg" "$out" --force) >> "$log" 2>&1; then - echo "FAIL dataset=$dataset variant=$variant seed=$seed log=$log" | tee -a "$log" - touch "$PAPER/exp/integrated/FAILED" - exit 1 + local cfg="" + { + flock 9 + if [[ -s "$QUEUE" ]]; then + cfg="$(head -n 1 "$QUEUE")" + tail -n +2 "$QUEUE" > "$QUEUE.tmp" + mv "$QUEUE.tmp" "$QUEUE" + fi + } 9>"$LOCK" + [[ -n "$cfg" ]] || break + local out="${cfg%.toml}" + local safe + safe="$(echo "$cfg" | tr '/ ' '__')" + local log="$LOG_ROOT/gpu${gpu}_${safe}.out" + echo "$gpu,$cfg,$(date -Is)" >> "$USAGE_CSV" + if [[ "$FORCE" != "1" ]] && check_done "$out" >/dev/null 2>&1; then + echo "SKIP $cfg" | tee -a "$log" + cleanup_artifacts "$out" + continue fi - if [[ ! -f "$out/DONE" || ! -f "$out/report.json" ]]; then - echo "FAIL missing DONE/report dataset=$dataset variant=$variant seed=$seed log=$log" | tee -a "$log" - touch "$PAPER/exp/integrated/FAILED" - exit 1 + mkdir -p "$out" + echo "RUN gpu=$gpu cfg=$cfg out=$out" | tee "$log" + set +e + "$PYTHON" bin/run_integrated.py "$cfg" --output "$out" --force >> "$log" 2>&1 + rc=$? + set -e + cleanup_artifacts "$out" + if [[ $rc -ne 0 ]]; then + echo "FAILED rc=$rc cfg=$cfg log=$log" | tee -a "$LOG_ROOT/FAILED" + return $rc fi - if ! "$PYTHON" - "$out/report.json" <<'PYCHK' -import json, sys -r=json.load(open(sys.argv[1])) -sys.exit(1 if r.get('failure') else 0) -PYCHK - then - echo "FAIL failure block dataset=$dataset variant=$variant seed=$seed log=$log" | tee -a "$log" - touch "$PAPER/exp/integrated/FAILED" - exit 1 + if ! check_done "$out" >/dev/null 2>&1; then + echo "FAILED missing DONE/report or failure block cfg=$cfg log=$log" | tee -a "$LOG_ROOT/FAILED" + return 1 fi - echo "DONE $(date -Iseconds) gpu=$gpu dataset=$dataset variant=$variant seed=$seed" | tee -a "$log" + echo "DONE $cfg" | tee -a "$log" done } -rm -f "$PAPER/exp/integrated/FAILED" + pids=() -for gpu in $(seq 0 $((N_GPUS - 1))); do worker "$gpu" & pids+=("$!"); done -status=0 -for pid in "${pids[@]}"; do wait "$pid" || status=1; done -"$PYTHON" "$ROOT/tools/aggregate_integrated_results.py" || status=1 -exit "$status" +for ((gpu=0; gpu&2 + exit $rc +fi +"$PYTHON" ../tools/aggregate_integrated_results.py --stage wave --manifest "$MANIFEST" || true +used_gpus=$(tail -n +2 "$USAGE_CSV" | cut -d, -f1 | sort -n | uniq | tr '\n' ' ') +echo "used_gpus: $used_gpus" +echo "logs: $LOG_ROOT" From f46c761f8ea793e0786ed1d8da299ed13a097a18 Mon Sep 17 00:00:00 2001 From: abhipray-chavan <53275745+abhipray-chavan@users.noreply.github.com> Date: Sun, 26 Apr 2026 18:42:06 +0000 Subject: [PATCH 3/7] Improve integrated TabM results --- tabm_integrated/FINAL_EXPERIMENT_REPORT.md | 94 +++++++++------- .../paper/exp/final_integrated_summary.csv | 62 +++++------ .../tools/aggregate_integrated_results.py | 100 ++++++++++++++---- 3 files changed, 171 insertions(+), 85 deletions(-) diff --git a/tabm_integrated/FINAL_EXPERIMENT_REPORT.md b/tabm_integrated/FINAL_EXPERIMENT_REPORT.md index ad80ccbb3..82f6fa18f 100644 --- a/tabm_integrated/FINAL_EXPERIMENT_REPORT.md +++ b/tabm_integrated/FINAL_EXPERIMENT_REPORT.md @@ -1,41 +1,63 @@ # Integrated TabM final experiment report Official TabM PLR/PiecewiseLinearEmbeddings baseline configs are preserved exactly; variants add only module flags for RLA, ESAM, MFB, and CF-FISD. +Deltas and win/loss statuses are computed against `baseline_plr` using the same selected inference mode (`mean`, `best-head`, or `greedy-heads`) as the variant row. + +## Matched-inference correction + +The earlier mean-baseline comparison changed when every variant was compared with the matching baseline inference mode. Per the rescue protocol stop condition, no new rescue sweep is claimed in this report. + +| dataset | variant | inference | mean-baseline status | matched-baseline status | mean-baseline delta | matched delta | +|---|---|---|---|---|---:|---:| +| sberbank-housing | best_esam_only | greedy-heads | weak_win | loss | 1.17846e-05 | -0.000301017 | +| ecom-offers | best_rla_only | best-head | clear_win | loss | 0.00838888 | -0.00124693 | +| ecom-offers | best_esam_only | greedy-heads | clear_win | weak_win | 0.0103616 | 4.90059e-05 | +| ecom-offers | best_cf_fisd_only | greedy-heads | clear_win | loss | 0.0101468 | -0.000165703 | +| homesite-insurance | best_rla_only | greedy-heads | weak_win | loss | 6.98613e-05 | -0.000111505 | +| homesite-insurance | best_esam_only | greedy-heads | clear_win | weak_win | 0.000237685 | 5.63183e-05 | +| homesite-insurance | best_cf_fisd_only | greedy-heads | clear_win | weak_win | 0.000413458 | 0.000232091 | +| homesite-insurance | best_combined | greedy-heads | clear_win | weak_win | 0.000354836 | 0.000173469 | +| cooking-time | best_rla_only | greedy-heads | clear_win | loss | 0.000234391 | -2.08739e-05 | +| cooking-time | best_esam_only | greedy-heads | clear_win | weak_win | 0.000315778 | 6.0513e-05 | +| cooking-time | best_mfb_only | greedy-heads | clear_win | weak_win | 0.000407806 | 0.000152541 | +| cooking-time | best_cf_fisd_only | greedy-heads | weak_win | loss | 4.32134e-05 | -0.000212051 | +| cooking-time | best_combined | greedy-heads | clear_win | weak_win | 0.000354592 | 9.93268e-05 | +| delivery-eta | best_esam_only | greedy-heads | loss | weak_win | -0.000922076 | 2.65273e-05 | ## Final 3-seed results -| dataset | variant | source config | metric | inference | mean ± std | delta | status | -|---|---|---|---|---|---:|---:|---| -| sberbank-housing | baseline_plr | baseline_plr | RMSE | mean | 0.234418 ± 0.00354822 | 0 | baseline | -| sberbank-housing | best_rla_only | rla_rank2_noise0.001 | RMSE | greedy-heads | 0.232368 ± 0.00228551 | 0.0020499 | weak_win | -| sberbank-housing | best_esam_only | esam_rho0.0025 | RMSE | greedy-heads | 0.234406 ± 0.00401172 | 1.17846e-05 | weak_win | -| sberbank-housing | best_mfb_only | mfb_keep0.8 | RMSE | greedy-heads | 0.2351 ± 0.00395661 | -0.000681772 | loss | -| sberbank-housing | best_cf_fisd_only | cf_fisd_only | RMSE | greedy-heads | 0.233093 ± 0.000585082 | 0.0013248 | weak_win | -| sberbank-housing | best_combined | mfb_cf_fisd | RMSE | greedy-heads | 0.234979 ± 0.00337063 | -0.000560913 | loss | -| ecom-offers | baseline_plr | baseline_plr | AUROC | mean | 0.590058 ± 0.000970364 | 0 | baseline | -| ecom-offers | best_rla_only | rla_rank2_noise0.0001 | AUROC | best-head | 0.598447 ± 0.00279735 | 0.00838888 | clear_win | -| ecom-offers | best_esam_only | esam_rho0.001 | AUROC | greedy-heads | 0.60042 ± 0.00349335 | 0.0103616 | clear_win | -| ecom-offers | best_mfb_only | mfb_keep0.7 | AUROC | best-head | 0.609516 ± 0.00208948 | 0.0194584 | clear_win | -| ecom-offers | best_cf_fisd_only | cf_fisd_lambda0.05 | AUROC | greedy-heads | 0.600205 ± 0.00345825 | 0.0101468 | clear_win | -| ecom-offers | best_combined | mfb_cf_fisd | AUROC | greedy-heads | 0.605211 ± 0.00322248 | 0.0151529 | clear_win | -| homesite-insurance | baseline_plr | baseline_plr | AUROC | mean | 0.962494 ± 0.000178079 | 0 | baseline | -| homesite-insurance | best_rla_only | rla_rank4_noise1e-05 | AUROC | greedy-heads | 0.962564 ± 0.000608529 | 6.98613e-05 | weak_win | -| homesite-insurance | best_esam_only | esam_only | AUROC | greedy-heads | 0.962732 ± 0.00044923 | 0.000237685 | clear_win | -| homesite-insurance | best_mfb_only | mfb_keep0.95 | AUROC | greedy-heads | 0.962382 ± 0.00045772 | -0.000111855 | loss | -| homesite-insurance | best_cf_fisd_only | cf_fisd_only | AUROC | greedy-heads | 0.962908 ± 0.000474027 | 0.000413458 | clear_win | -| homesite-insurance | best_combined | mfb_cf_fisd | AUROC | greedy-heads | 0.962849 ± 0.000437968 | 0.000354836 | clear_win | -| cooking-time | baseline_plr | baseline_plr | RMSE | mean | 0.480381 ± 0.000123385 | 0 | baseline | -| cooking-time | best_rla_only | rla_rank4_noise0.001 | RMSE | greedy-heads | 0.480146 ± 0.00016089 | 0.000234391 | clear_win | -| cooking-time | best_esam_only | esam_rho0.0025 | RMSE | greedy-heads | 0.480065 ± 8.02121e-05 | 0.000315778 | clear_win | -| cooking-time | best_mfb_only | mfb_keep0.8 | RMSE | greedy-heads | 0.479973 ± 0.000194552 | 0.000407806 | clear_win | -| cooking-time | best_cf_fisd_only | cf_fisd_lambda0.05 | RMSE | greedy-heads | 0.480337 ± 4.64953e-05 | 4.32134e-05 | weak_win | -| cooking-time | best_combined | rla_esam | RMSE | greedy-heads | 0.480026 ± 9.6554e-05 | 0.000354592 | clear_win | -| delivery-eta | baseline_plr | baseline_plr | RMSE | mean | 0.550226 ± 0.000582292 | 0 | baseline | -| delivery-eta | best_rla_only | rla_rank2_noise0.001 | RMSE | greedy-heads | 0.551407 ± 0.00109716 | -0.00118071 | loss | -| delivery-eta | best_esam_only | esam_only | RMSE | greedy-heads | 0.551148 ± 0.000504719 | -0.000922076 | loss | -| delivery-eta | best_mfb_only | mfb_keep0.7 | RMSE | greedy-heads | 0.551366 ± 0.00171387 | -0.00114032 | loss | -| delivery-eta | best_cf_fisd_only | cf_fisd_lambda0.05 | RMSE | greedy-heads | 0.552434 ± 0.00139117 | -0.00220824 | loss | -| delivery-eta | best_combined | rla_esam | RMSE | greedy-heads | 0.552336 ± 0.000198974 | -0.00211043 | loss | +| dataset | task | variant | selected config | metric | direction | inference | matched baseline mean ± std | result mean ± std | delta | % delta | n | status | config path | result path | +|---|---|---|---|---|---|---|---:|---:|---:|---:|---:|---|---|---| +| sberbank-housing | regression | baseline_plr | baseline_plr | RMSE | lower | mean | 0.234418 ± 0.00354822 | 0.234418 ± 0.00354822 | 0 | 0 | 3 | baseline | `exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2` | +| sberbank-housing | regression | best_rla_only | rla_rank2_noise0.001 | RMSE | lower | greedy-heads | 0.234105 ± 0.00278385 | 0.232368 ± 0.00228551 | 0.0017371 | 0.742018 | 3 | weak_win | `exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2` | +| sberbank-housing | regression | best_esam_only | esam_rho0.0025 | RMSE | lower | greedy-heads | 0.234105 ± 0.00278385 | 0.234406 ± 0.00401172 | -0.000301017 | -0.128582 | 3 | loss | `exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2` | +| sberbank-housing | regression | best_mfb_only | mfb_keep0.8 | RMSE | lower | greedy-heads | 0.234105 ± 0.00278385 | 0.2351 ± 0.00395661 | -0.000994573 | -0.424841 | 3 | loss | `exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2` | +| sberbank-housing | regression | best_cf_fisd_only | cf_fisd_only | RMSE | lower | greedy-heads | 0.234105 ± 0.00278385 | 0.233093 ± 0.000585082 | 0.001012 | 0.432283 | 3 | weak_win | `exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2` | +| sberbank-housing | regression | best_combined | mfb_cf_fisd | RMSE | lower | greedy-heads | 0.234105 ± 0.00278385 | 0.234979 ± 0.00337063 | -0.000873715 | -0.373215 | 3 | loss | `exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2` | +| ecom-offers | binclass | baseline_plr | baseline_plr | AUROC | higher | mean | 0.590058 ± 0.000970364 | 0.590058 ± 0.000970364 | 0 | 0 | 3 | baseline | `exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml` | `exp/integrated/final/ecom-offers/baseline_plr-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2` | +| ecom-offers | binclass | best_rla_only | rla_rank2_noise0.0001 | AUROC | higher | best-head | 0.599694 ± 0.00357854 | 0.598447 ± 0.00279735 | -0.00124693 | -0.207927 | 3 | loss | `exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2` | +| ecom-offers | binclass | best_esam_only | esam_rho0.001 | AUROC | higher | greedy-heads | 0.600371 ± 0.00344598 | 0.60042 ± 0.00349335 | 4.90059e-05 | 0.0081626 | 3 | weak_win | `exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2` | +| ecom-offers | binclass | best_mfb_only | mfb_keep0.7 | AUROC | higher | best-head | 0.599694 ± 0.00357854 | 0.609516 ± 0.00208948 | 0.00982263 | 1.63794 | 3 | clear_win | `exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2` | +| ecom-offers | binclass | best_cf_fisd_only | cf_fisd_lambda0.05 | AUROC | higher | greedy-heads | 0.600371 ± 0.00344598 | 0.600205 ± 0.00345825 | -0.000165703 | -0.0276002 | 3 | loss | `exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2` | +| ecom-offers | binclass | best_combined | mfb_cf_fisd | AUROC | higher | greedy-heads | 0.600371 ± 0.00344598 | 0.605211 ± 0.00322248 | 0.00484036 | 0.806228 | 3 | clear_win | `exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2` | +| homesite-insurance | binclass | baseline_plr | baseline_plr | AUROC | higher | mean | 0.962494 ± 0.000178079 | 0.962494 ± 0.000178079 | 0 | 0 | 3 | baseline | `exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2` | +| homesite-insurance | binclass | best_rla_only | rla_rank4_noise1e-05 | AUROC | higher | greedy-heads | 0.962676 ± 0.000276725 | 0.962564 ± 0.000608529 | -0.000111505 | -0.0115828 | 3 | loss | `exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2` | +| homesite-insurance | binclass | best_esam_only | esam_only | AUROC | higher | greedy-heads | 0.962676 ± 0.000276725 | 0.962732 ± 0.00044923 | 5.63183e-05 | 0.00585019 | 3 | weak_win | `exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2` | +| homesite-insurance | binclass | best_mfb_only | mfb_keep0.95 | AUROC | higher | greedy-heads | 0.962676 ± 0.000276725 | 0.962382 ± 0.00045772 | -0.000293222 | -0.030459 | 3 | loss | `exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2` | +| homesite-insurance | binclass | best_cf_fisd_only | cf_fisd_only | AUROC | higher | greedy-heads | 0.962676 ± 0.000276725 | 0.962908 ± 0.000474027 | 0.000232091 | 0.024109 | 3 | weak_win | `exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2` | +| homesite-insurance | binclass | best_combined | mfb_cf_fisd | AUROC | higher | greedy-heads | 0.962676 ± 0.000276725 | 0.962849 ± 0.000437968 | 0.000173469 | 0.0180195 | 3 | weak_win | `exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2` | +| cooking-time | regression | baseline_plr | baseline_plr | RMSE | lower | mean | 0.480381 ± 0.000123385 | 0.480381 ± 0.000123385 | 0 | 0 | 3 | baseline | `exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml` | `exp/integrated/final/cooking-time/baseline_plr-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-evaluation/2` | +| cooking-time | regression | best_rla_only | rla_rank4_noise0.001 | RMSE | lower | greedy-heads | 0.480125 ± 0.000258675 | 0.480146 ± 0.00016089 | -2.08739e-05 | -0.00434759 | 3 | loss | `exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2` | +| cooking-time | regression | best_esam_only | esam_rho0.0025 | RMSE | lower | greedy-heads | 0.480125 ± 0.000258675 | 0.480065 ± 8.02121e-05 | 6.0513e-05 | 0.0126036 | 3 | weak_win | `exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2` | +| cooking-time | regression | best_mfb_only | mfb_keep0.8 | RMSE | lower | greedy-heads | 0.480125 ± 0.000258675 | 0.479973 ± 0.000194552 | 0.000152541 | 0.031771 | 3 | weak_win | `exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2` | +| cooking-time | regression | best_cf_fisd_only | cf_fisd_lambda0.05 | RMSE | lower | greedy-heads | 0.480125 ± 0.000258675 | 0.480337 ± 4.64953e-05 | -0.000212051 | -0.0441658 | 3 | loss | `exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2` | +| cooking-time | regression | best_combined | rla_esam | RMSE | lower | greedy-heads | 0.480125 ± 0.000258675 | 0.480026 ± 9.6554e-05 | 9.93268e-05 | 0.0206877 | 3 | weak_win | `exp/integrated/final/cooking-time/best_combined-evaluation/0.toml;exp/integrated/final/cooking-time/best_combined-evaluation/1.toml;exp/integrated/final/cooking-time/best_combined-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2` | +| delivery-eta | regression | baseline_plr | baseline_plr | RMSE | lower | mean | 0.550226 ± 0.000582292 | 0.550226 ± 0.000582292 | 0 | 0 | 3 | baseline | `exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml` | `exp/integrated/final/delivery-eta/baseline_plr-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2` | +| delivery-eta | regression | best_rla_only | rla_rank2_noise0.001 | RMSE | lower | greedy-heads | 0.551174 ± 0.000706865 | 0.551407 ± 0.00109716 | -0.00023211 | -0.0421119 | 3 | loss | `exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2` | +| delivery-eta | regression | best_esam_only | esam_only | RMSE | lower | greedy-heads | 0.551174 ± 0.000706865 | 0.551148 ± 0.000504719 | 2.65273e-05 | 0.00481288 | 3 | weak_win | `exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2` | +| delivery-eta | regression | best_mfb_only | mfb_keep0.7 | RMSE | lower | greedy-heads | 0.551174 ± 0.000706865 | 0.551366 ± 0.00171387 | -0.000191714 | -0.0347828 | 3 | loss | `exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2` | +| delivery-eta | regression | best_cf_fisd_only | cf_fisd_lambda0.05 | RMSE | lower | greedy-heads | 0.551174 ± 0.000706865 | 0.552434 ± 0.00139117 | -0.00125964 | -0.228537 | 3 | loss | `exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2` | +| delivery-eta | regression | best_combined | rla_esam | RMSE | lower | greedy-heads | 0.551174 ± 0.000706865 | 0.552336 ± 0.000198974 | -0.00116182 | -0.210791 | 3 | loss | `exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2` | ## Validation-selected configs @@ -72,10 +94,10 @@ Official TabM PLR/PiecewiseLinearEmbeddings baseline configs are preserved exact | delivery-eta | best_cf_fisd_only | cf_fisd_lambda0.05 | greedy-heads | 0.5556691040453957 | | delivery-eta | best_combined | rla_esam | greedy-heads | 0.5552988041877751 | -## Module wins vs baseline +## Module wins vs matched baseline -- `best_rla_only`: sberbank-housing, ecom-offers, homesite-insurance, cooking-time -- `best_esam_only`: sberbank-housing, ecom-offers, homesite-insurance, cooking-time +- `best_rla_only`: sberbank-housing +- `best_esam_only`: ecom-offers, homesite-insurance, cooking-time, delivery-eta - `best_mfb_only`: ecom-offers, cooking-time -- `best_cf_fisd_only`: sberbank-housing, ecom-offers, homesite-insurance, cooking-time +- `best_cf_fisd_only`: sberbank-housing, homesite-insurance - `best_combined`: ecom-offers, homesite-insurance, cooking-time diff --git a/tabm_integrated/paper/exp/final_integrated_summary.csv b/tabm_integrated/paper/exp/final_integrated_summary.csv index 2de242084..d76516a4b 100644 --- a/tabm_integrated/paper/exp/final_integrated_summary.csv +++ b/tabm_integrated/paper/exp/final_integrated_summary.csv @@ -1,31 +1,31 @@ -dataset,variant,source_variant,metric,direction,validation_metric,test_metric,mean,std,n_seeds,baseline_mean,absolute_delta,percent_delta,precision,inference_mode,config_path,result_path,status -sberbank-housing,baseline_plr,baseline_plr,RMSE,lower,0.23552481981288087,0.23441786078206484,0.23441786078206484,0.0035482159973780638,3,0.23441786078206484,0.0,0.0,3 seeds,mean,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2,baseline -sberbank-housing,best_rla_only,rla_rank2_noise0.001,RMSE,lower,0.23385412997377952,0.2323679570932143,0.2323679570932143,0.002285510337847783,3,0.23441786078206484,0.002049903688850524,0.8744656580397225,3 seeds,greedy-heads,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2,weak_win -sberbank-housing,best_esam_only,esam_rho0.0025,RMSE,lower,0.23381482399811443,0.23440607616465495,0.23440607616465495,0.004011724082624796,3,0.23441786078206484,1.1784617409887899e-05,0.005027184093640331,3 seeds,greedy-heads,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2,weak_win -sberbank-housing,best_mfb_only,mfb_keep0.8,RMSE,lower,0.23321906081613947,0.23509963234240183,0.23509963234240183,0.003956610803973734,3,0.23441786078206484,-0.0006817715603369967,-0.29083601311882573,3 seeds,greedy-heads,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2,loss -sberbank-housing,best_cf_fisd_only,cf_fisd_only,RMSE,lower,0.23407921163352594,0.23309306252476805,0.23309306252476805,0.0005850816922811394,3,0.23441786078206484,0.0013247982572967831,0.5651439070713261,3 seeds,greedy-heads,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2,weak_win -sberbank-housing,best_combined,mfb_cf_fisd,RMSE,lower,0.23329192884803412,0.23497877374993395,0.23497877374993395,0.003370630932382365,3,0.23441786078206484,-0.0005609129678691149,-0.2392791086813083,3 seeds,greedy-heads,exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2,loss -ecom-offers,baseline_plr,baseline_plr,AUROC,higher,0.6214296509780997,0.5900580480375381,0.5900580480375381,0.0009703638630149601,3,0.5900580480375381,0.0,0.0,3 seeds,mean,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2,baseline -ecom-offers,best_rla_only,rla_rank2_noise0.0001,AUROC,higher,0.6348279404183594,0.5984469268269463,0.5984469268269463,0.002797349140724218,3,0.5900580480375381,0.008388878789408194,1.4217039861262113,3 seeds,best-head,exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2,clear_win -ecom-offers,best_esam_only,esam_rho0.001,AUROC,higher,0.6366563506545114,0.6004196058355985,0.6004196058355985,0.003493353451214683,3,0.5900580480375381,0.010361557798060361,1.7560234679488995,3 seeds,greedy-heads,exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2,clear_win -ecom-offers,best_mfb_only,mfb_keep0.7,AUROC,higher,0.6539769427777475,0.6095164790194458,0.6095164790194458,0.0020894834660806928,3,0.5900580480375381,0.019458430981907693,3.2977146988544748,3 seeds,best-head,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2,clear_win -ecom-offers,best_cf_fisd_only,cf_fisd_lambda0.05,AUROC,higher,0.636440295793426,0.6002048966489212,0.6002048966489212,0.0034582462660955543,3,0.5900580480375381,0.010146848611383041,1.719635660445652,3 seeds,greedy-heads,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2,clear_win -ecom-offers,best_combined,mfb_cf_fisd,AUROC,higher,0.6485201184226032,0.6052109584242933,0.6052109584242933,0.003222479623952224,3,0.5900580480375381,0.015152910386755147,2.5680372358536414,3 seeds,greedy-heads,exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml,exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2,clear_win -homesite-insurance,baseline_plr,baseline_plr,AUROC,higher,0.9584198249724393,0.9624941853379376,0.9624941853379376,0.0001780794332642721,3,0.9624941853379376,0.0,0.0,3 seeds,mean,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2,baseline -homesite-insurance,best_rla_only,rla_rank4_noise1e-05,AUROC,higher,0.9589784960899528,0.9625640466562138,0.9625640466562138,0.0006085287596608143,3,0.9624941853379376,6.986131827613207e-05,0.007258362631209385,3 seeds,greedy-heads,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2,weak_win -homesite-insurance,best_esam_only,esam_only,AUROC,higher,0.959080906574986,0.9627318700697393,0.9627318700697393,0.0004492301463849077,3,0.9624941853379376,0.00023768473180163507,0.024694666775382387,3 seeds,greedy-heads,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2,clear_win -homesite-insurance,best_mfb_only,mfb_keep0.95,AUROC,higher,0.959182264937648,0.962382329909275,0.962382329909275,0.0004577202193422504,3,0.9624941853379376,-0.00011185542866265497,-0.011621413445046615,3 seeds,greedy-heads,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2,loss -homesite-insurance,best_cf_fisd_only,cf_fisd_only,AUROC,higher,0.959527988245724,0.9629076428862089,0.9629076428862089,0.00047402677133884024,3,0.9624941853379376,0.0004134575482712677,0.04295688790328642,3 seeds,greedy-heads,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2,clear_win -homesite-insurance,best_combined,mfb_cf_fisd,AUROC,higher,0.95963296602657,0.9628490210733115,0.9628490210733115,0.0004379684704485265,3,0.9624941853379376,0.0003548357353738707,0.03686627314525393,3 seeds,greedy-heads,exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2,clear_win -cooking-time,baseline_plr,baseline_plr,RMSE,lower,0.46270875116661525,0.4803806470025843,0.4803806470025843,0.00012338540954176603,3,0.4803806470025843,0.0,0.0,3 seeds,mean,exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml,exp/integrated/final/cooking-time/baseline_plr-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-evaluation/2,baseline -cooking-time,best_rla_only,rla_rank4_noise0.001,RMSE,lower,0.46244170136784024,0.48014625604758127,0.48014625604758127,0.00016088993704992786,3,0.4803806470025843,0.00023439095500304985,0.04879275559196058,3 seeds,greedy-heads,exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2,clear_win -cooking-time,best_esam_only,esam_rho0.0025,RMSE,lower,0.46247755253655587,0.4800648691653293,0.4800648691653293,8.021206715574865e-05,3,0.4803806470025843,0.00031577783725500463,0.0657349206770409,3 seeds,greedy-heads,exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2,clear_win -cooking-time,best_mfb_only,mfb_keep0.8,RMSE,lower,0.4625031808631155,0.4799728414668436,0.4799728414668436,0.00019455249694504936,3,0.4803806470025843,0.0004078055357407129,0.0848921658866326,3 seeds,greedy-heads,exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2,clear_win -cooking-time,best_cf_fisd_only,cf_fisd_lambda0.05,RMSE,lower,0.46295431080753496,0.48033743363756964,0.48033743363756964,4.649528370692734e-05,3,0.4803806470025843,4.3213365014671545e-05,0.008995650695819781,3 seeds,greedy-heads,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2,weak_win -cooking-time,best_combined,rla_esam,RMSE,lower,0.46244255150399494,0.48002605538016,0.48002605538016,9.65539747406379e-05,3,0.4803806470025843,0.000354591622424294,0.07381471852307706,3 seeds,greedy-heads,exp/integrated/final/cooking-time/best_combined-evaluation/0.toml;exp/integrated/final/cooking-time/best_combined-evaluation/1.toml;exp/integrated/final/cooking-time/best_combined-evaluation/2.toml,exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2,clear_win -delivery-eta,baseline_plr,baseline_plr,RMSE,lower,0.5559497041253089,0.5502258480608934,0.5502258480608934,0.0005822921162605647,3,0.5502258480608934,0.0,0.0,3 seeds,mean,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2,baseline -delivery-eta,best_rla_only,rla_rank2_noise0.001,RMSE,lower,0.5556523766610589,0.5514065614560619,0.5514065614560619,0.001097158835638919,3,0.5502258480608934,-0.0011807133951685644,-0.21458704627011546,3 seeds,greedy-heads,exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2,loss -delivery-eta,best_esam_only,esam_only,RMSE,lower,0.5554227463560124,0.5511479242060323,0.5511479242060323,0.0005047191238214393,3,0.5502258480608934,-0.0009220761451389503,-0.16758139378375847,3 seeds,greedy-heads,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2,loss -delivery-eta,best_mfb_only,mfb_keep0.7,RMSE,lower,0.5550787103394562,0.551366165184366,0.551366165184366,0.0017138685603590752,3,0.5502258480608934,-0.0011403171234726228,-0.20724528436665232,3 seeds,greedy-heads,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2,loss -delivery-eta,best_cf_fisd_only,cf_fisd_lambda0.05,RMSE,lower,0.5558348472452875,0.5524340881807381,0.5524340881807381,0.0013911659814170834,3,0.5502258480608934,-0.002208240119844773,-0.40133340293391445,3 seeds,greedy-heads,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2,loss -delivery-eta,best_combined,rla_esam,RMSE,lower,0.5558806192547581,0.5523362750771902,0.5523362750771902,0.00019897426137676394,3,0.5502258480608934,-0.002110427016296823,-0.3835565020680858,3 seeds,greedy-heads,exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml,exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2,loss +dataset,task_type,variant,source_variant,selected_config,metric,direction,metric_direction,validation_metric,test_metric,mean,std,test_mean,test_std,n_seeds,baseline_mean,baseline_std,matched_baseline_inference_mode,matched_baseline_mean,matched_baseline_std,absolute_delta,percent_delta,precision,inference_mode,selected_inference_mode,config_path,result_path,matched_baseline_result_path,status +sberbank-housing,regression,baseline_plr,baseline_plr,baseline_plr,RMSE,lower,lower,0.23552481981288087,0.23441786078206484,0.23441786078206484,0.0035482159973780638,0.23441786078206484,0.0035482159973780638,3,0.23441786078206484,0.0035482159973780638,mean,0.23441786078206484,0.0035482159973780638,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2,baseline +sberbank-housing,regression,best_rla_only,rla_rank2_noise0.001,rla_rank2_noise0.001,RMSE,lower,lower,0.23385412997377952,0.2323679570932143,0.2323679570932143,0.002285510337847783,0.2323679570932143,0.002285510337847783,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,0.0017371018906270908,0.7420180914360294,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,weak_win +sberbank-housing,regression,best_esam_only,esam_rho0.0025,esam_rho0.0025,RMSE,lower,lower,0.23381482399811443,0.23440607616465495,0.23440607616465495,0.004011724082624796,0.23440607616465495,0.004011724082624796,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,-0.00030101718081354534,-0.12858209135682216,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,loss +sberbank-housing,regression,best_mfb_only,mfb_keep0.8,mfb_keep0.8,RMSE,lower,lower,0.23321906081613947,0.23509963234240183,0.23509963234240183,0.003956610803973734,0.23509963234240183,0.003956610803973734,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,-0.00099457335856043,-0.4248406091169,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,loss +sberbank-housing,regression,best_cf_fisd_only,cf_fisd_only,cf_fisd_only,RMSE,lower,lower,0.23407921163352594,0.23309306252476805,0.23309306252476805,0.0005850816922811394,0.23309306252476805,0.0005850816922811394,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,0.00101199645907335,0.4322830371398342,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,weak_win +sberbank-housing,regression,best_combined,mfb_cf_fisd,mfb_cf_fisd,RMSE,lower,lower,0.23329192884803412,0.23497877374993395,0.23497877374993395,0.003370630932382365,0.23497877374993395,0.003370630932382365,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,-0.0008737147660925482,-0.3732148164097788,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,loss +ecom-offers,binclass,baseline_plr,baseline_plr,baseline_plr,AUROC,higher,higher,0.6214296509780997,0.5900580480375381,0.5900580480375381,0.0009703638630149601,0.5900580480375381,0.0009703638630149601,3,0.5900580480375381,0.0009703638630149601,mean,0.5900580480375381,0.0009703638630149601,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2,baseline +ecom-offers,binclass,best_rla_only,rla_rank2_noise0.0001,rla_rank2_noise0.0001,AUROC,higher,higher,0.6348279404183594,0.5984469268269463,0.5984469268269463,0.002797349140724218,0.5984469268269463,0.002797349140724218,3,0.5996938524587336,0.0035785432553676923,best-head,0.5996938524587336,0.0035785432553676923,-0.001246925631787299,-0.20792703254751188,3 seeds,best-head,best-head,exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2,loss +ecom-offers,binclass,best_esam_only,esam_rho0.001,esam_rho0.001,AUROC,higher,higher,0.6366563506545114,0.6004196058355985,0.6004196058355985,0.003493353451214683,0.6004196058355985,0.003493353451214683,3,0.6003705999706029,0.0034459818868121214,greedy-heads,0.6003705999706029,0.0034459818868121214,4.900586499556603e-05,0.008162602398912538,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2,weak_win +ecom-offers,binclass,best_mfb_only,mfb_keep0.7,mfb_keep0.7,AUROC,higher,higher,0.6539769427777475,0.6095164790194458,0.6095164790194458,0.0020894834660806928,0.6095164790194458,0.0020894834660806928,3,0.5996938524587336,0.0035785432553676923,best-head,0.5996938524587336,0.0035785432553676923,0.0098226265607122,1.637940179049629,3 seeds,best-head,best-head,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2,clear_win +ecom-offers,binclass,best_cf_fisd_only,cf_fisd_lambda0.05,cf_fisd_lambda0.05,AUROC,higher,higher,0.636440295793426,0.6002048966489212,0.6002048966489212,0.0034582462660955543,0.6002048966489212,0.0034582462660955543,3,0.6003705999706029,0.0034459818868121214,greedy-heads,0.6003705999706029,0.0034459818868121214,-0.00016570332168175383,-0.02760017257505072,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2,loss +ecom-offers,binclass,best_combined,mfb_cf_fisd,mfb_cf_fisd,AUROC,higher,higher,0.6485201184226032,0.6052109584242933,0.6052109584242933,0.003222479623952224,0.6052109584242933,0.003222479623952224,3,0.6003705999706029,0.0034459818868121214,greedy-heads,0.6003705999706029,0.0034459818868121214,0.004840358453690352,0.8062284285618516,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml,exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2,clear_win +homesite-insurance,binclass,baseline_plr,baseline_plr,baseline_plr,AUROC,higher,higher,0.9584198249724393,0.9624941853379376,0.9624941853379376,0.0001780794332642721,0.9624941853379376,0.0001780794332642721,3,0.9624941853379376,0.0001780794332642721,mean,0.9624941853379376,0.0001780794332642721,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2,baseline +homesite-insurance,binclass,best_rla_only,rla_rank4_noise1e-05,rla_rank4_noise1e-05,AUROC,higher,higher,0.9589784960899528,0.9625640466562138,0.9625640466562138,0.0006085287596608143,0.9625640466562138,0.0006085287596608143,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,-0.00011150507503732143,-0.011582830252289419,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,loss +homesite-insurance,binclass,best_esam_only,esam_only,esam_only,AUROC,higher,higher,0.959080906574986,0.9627318700697393,0.9627318700697393,0.0004492301463849077,0.9627318700697393,0.0004492301463849077,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,5.6318338488181574e-05,0.005850188922623008,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,weak_win +homesite-insurance,binclass,best_mfb_only,mfb_keep0.95,mfb_keep0.95,AUROC,higher,higher,0.959182264937648,0.962382329909275,0.962382329909275,0.0004577202193422504,0.962382329909275,0.0004577202193422504,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,-0.00029322182197610847,-0.030459049411692845,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,loss +homesite-insurance,binclass,best_cf_fisd_only,cf_fisd_only,cf_fisd_only,AUROC,higher,higher,0.959527988245724,0.9629076428862089,0.9629076428862089,0.00047402677133884024,0.9629076428862089,0.00047402677133884024,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,0.00023209115495781418,0.024108969479948606,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,weak_win +homesite-insurance,binclass,best_combined,mfb_cf_fisd,mfb_cf_fisd,AUROC,higher,higher,0.95963296602657,0.9628490210733115,0.9628490210733115,0.0004379684704485265,0.9628490210733115,0.0004379684704485265,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,0.0001734693420604172,0.018019502183103577,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,weak_win +cooking-time,regression,baseline_plr,baseline_plr,baseline_plr,RMSE,lower,lower,0.46270875116661525,0.4803806470025843,0.4803806470025843,0.00012338540954176603,0.4803806470025843,0.00012338540954176603,3,0.4803806470025843,0.00012338540954176603,mean,0.4803806470025843,0.00012338540954176603,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml,exp/integrated/final/cooking-time/baseline_plr-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-evaluation/2,baseline +cooking-time,regression,best_rla_only,rla_rank4_noise0.001,rla_rank4_noise0.001,RMSE,lower,lower,0.46244170136784024,0.48014625604758127,0.48014625604758127,0.00016088993704992786,0.48014625604758127,0.00016088993704992786,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,-2.08738622038962e-05,-0.004347585647083486,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,loss +cooking-time,regression,best_esam_only,esam_rho0.0025,esam_rho0.0025,RMSE,lower,lower,0.46247755253655587,0.4800648691653293,0.4800648691653293,8.021206715574865e-05,0.4800648691653293,8.021206715574865e-05,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,6.0513020048058586e-05,0.012603586957353233,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,weak_win +cooking-time,regression,best_mfb_only,mfb_keep0.8,mfb_keep0.8,RMSE,lower,lower,0.4625031808631155,0.4799728414668436,0.4799728414668436,0.00019455249694504936,0.4799728414668436,0.00019455249694504936,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,0.00015254071853376683,0.03177101736205868,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,weak_win +cooking-time,regression,best_cf_fisd_only,cf_fisd_lambda0.05,cf_fisd_lambda0.05,RMSE,lower,lower,0.46295431080753496,0.48033743363756964,0.48033743363756964,4.649528370692734e-05,0.48033743363756964,4.649528370692734e-05,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,-0.0002120514521922745,-0.04416584918445345,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,loss +cooking-time,regression,best_combined,rla_esam,rla_esam,RMSE,lower,lower,0.46244255150399494,0.48002605538016,0.48002605538016,9.65539747406379e-05,0.48002605538016,9.65539747406379e-05,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,9.932680521734794e-05,0.020687680531540333,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_combined-evaluation/0.toml;exp/integrated/final/cooking-time/best_combined-evaluation/1.toml;exp/integrated/final/cooking-time/best_combined-evaluation/2.toml,exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,weak_win +delivery-eta,regression,baseline_plr,baseline_plr,baseline_plr,RMSE,lower,lower,0.5559497041253089,0.5502258480608934,0.5502258480608934,0.0005822921162605647,0.5502258480608934,0.0005822921162605647,3,0.5502258480608934,0.0005822921162605647,mean,0.5502258480608934,0.0005822921162605647,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2,baseline +delivery-eta,regression,best_rla_only,rla_rank2_noise0.001,rla_rank2_noise0.001,RMSE,lower,lower,0.5556523766610589,0.5514065614560619,0.5514065614560619,0.001097158835638919,0.5514065614560619,0.001097158835638919,3,0.5511744515458483,0.0007068650190236373,greedy-heads,0.5511744515458483,0.0007068650190236373,-0.00023210991021360705,-0.04211187756664361,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2,loss +delivery-eta,regression,best_esam_only,esam_only,esam_only,RMSE,lower,lower,0.5554227463560124,0.5511479242060323,0.5511479242060323,0.0005047191238214393,0.5511479242060323,0.0005047191238214393,3,0.5511744515458483,0.0007068650190236373,greedy-heads,0.5511744515458483,0.0007068650190236373,2.652733981600708e-05,0.004812875441088992,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2,weak_win +delivery-eta,regression,best_mfb_only,mfb_keep0.7,mfb_keep0.7,RMSE,lower,lower,0.5550787103394562,0.551366165184366,0.551366165184366,0.0017138685603590752,0.551366165184366,0.0017138685603590752,3,0.5511744515458483,0.0007068650190236373,greedy-heads,0.5511744515458483,0.0007068650190236373,-0.00019171363851766543,-0.03478275126504446,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2,loss +delivery-eta,regression,best_cf_fisd_only,cf_fisd_lambda0.05,cf_fisd_lambda0.05,RMSE,lower,lower,0.5558348472452875,0.5524340881807381,0.5524340881807381,0.0013911659814170834,0.5524340881807381,0.0013911659814170834,3,0.5511744515458483,0.0007068650190236373,greedy-heads,0.5511744515458483,0.0007068650190236373,-0.0012596366348898158,-0.22853683282252707,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2,loss +delivery-eta,regression,best_combined,rla_esam,rla_esam,RMSE,lower,lower,0.5558806192547581,0.5523362750771902,0.5523362750771902,0.00019897426137676394,0.5523362750771902,0.00019897426137676394,3,0.5511744515458483,0.0007068650190236373,greedy-heads,0.5511744515458483,0.0007068650190236373,-0.0011618235313418657,-0.21079052704336418,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml,exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2,loss diff --git a/tabm_integrated/tools/aggregate_integrated_results.py b/tabm_integrated/tools/aggregate_integrated_results.py index bfde966e3..c39da8918 100644 --- a/tabm_integrated/tools/aggregate_integrated_results.py +++ b/tabm_integrated/tools/aggregate_integrated_results.py @@ -196,6 +196,14 @@ def select_final_configs() -> None: print(f'final manifest: {FINAL_MANIFEST.relative_to(PAPER)} ({len(final_paths)} jobs)') +def task_type_from_metric(metric: str) -> str: + if metric == 'RMSE': + return 'regression' + if metric == 'AUROC': + return 'binclass' + return '' + + def summarize_final() -> list[dict[str, Any]]: selection = {} if SELECTED_PATH.exists(): @@ -205,7 +213,10 @@ def summarize_final() -> list[dict[str, Any]]: rows = [r for r in collect_rows() if r['wave'] == 'final'] wanted = set((d, v) for d in DATASET_ORDER for v in FINAL_DISPLAY) grouped = defaultdict(list) + baseline_grouped = defaultdict(list) for row in rows: + if row['variant'] == 'baseline_plr': + baseline_grouped[(row['dataset'], row['inference_mode'])].append(row) if (row['dataset'], row['variant']) not in wanted: continue sel = selection.get((row['dataset'], row['variant'])) @@ -216,11 +227,6 @@ def summarize_final() -> list[dict[str, Any]]: grouped[(row['dataset'], row['variant'])].append(row) out = [] - baseline_stats = {} - for dataset in DATASET_ORDER: - vals = [r['test_metric'] for r in grouped.get((dataset, 'baseline_plr'), [])] - baseline_stats[dataset] = (statistics.mean(vals), statistics.stdev(vals) if len(vals) > 1 else 0.0) if vals else (float('nan'), float('nan')) - for dataset in DATASET_ORDER: for variant in FINAL_DISPLAY: rs = sorted(grouped.get((dataset, variant), []), key=lambda r: r['seed']) @@ -228,37 +234,55 @@ def summarize_final() -> list[dict[str, Any]]: invalid = n != 3 or any(r['failure'] for r in rs) metric = rs[0]['metric'] if rs else '' direction = rs[0]['direction'] if rs else '' + selected_inference = 'mean' if variant == 'baseline_plr' else selection.get((dataset, variant), {}).get('inference_mode', rs[0]['inference_mode'] if rs else '') + baseline_rs = sorted(baseline_grouped.get((dataset, selected_inference), []), key=lambda r: r['seed']) + baseline_invalid = len(baseline_rs) != 3 or any(r['failure'] for r in baseline_rs) val_mean = statistics.mean([r['validation_metric'] for r in rs]) if rs else float('nan') test_values = [r['test_metric'] for r in rs] test_mean = statistics.mean(test_values) if test_values else float('nan') test_std = statistics.stdev(test_values) if len(test_values) > 1 else 0.0 - base_mean, base_std = baseline_stats[dataset] - delta, pct = signed_delta(test_mean, base_mean, direction) if rs else (float('nan'), float('nan')) - status = 'baseline' if variant == 'baseline_plr' else status_for(delta, base_std, n, invalid) + baseline_values = [r['test_metric'] for r in baseline_rs] + base_mean = statistics.mean(baseline_values) if baseline_values else float('nan') + base_std = statistics.stdev(baseline_values) if len(baseline_values) > 1 else 0.0 + delta, pct = signed_delta(test_mean, base_mean, direction) if rs and baseline_rs else (float('nan'), float('nan')) + status = 'baseline' if variant == 'baseline_plr' else status_for(delta, base_std, n, invalid or baseline_invalid) sel = selection.get((dataset, variant), {}) + source_variant = sel.get('source_variant', variant) + config_paths = ';'.join(r['config_path'] for r in rs) + result_paths = ';'.join(r['result_path'] for r in rs) out.append({ 'dataset': dataset, + 'task_type': task_type_from_metric(metric), 'variant': variant, - 'source_variant': sel.get('source_variant', variant), + 'source_variant': source_variant, + 'selected_config': source_variant, 'metric': metric, 'direction': direction, + 'metric_direction': direction, 'validation_metric': val_mean, 'test_metric': test_mean, 'mean': test_mean, 'std': test_std, + 'test_mean': test_mean, + 'test_std': test_std, 'n_seeds': n, 'baseline_mean': base_mean, + 'baseline_std': base_std, + 'matched_baseline_inference_mode': selected_inference, + 'matched_baseline_mean': base_mean, + 'matched_baseline_std': base_std, 'absolute_delta': delta, 'percent_delta': pct, 'precision': '3 seeds', - 'inference_mode': sel.get('inference_mode', 'mean' if variant == 'baseline_plr' else ''), - 'config_path': ';'.join(r['config_path'] for r in rs), - 'result_path': ';'.join(r['result_path'] for r in rs), + 'inference_mode': selected_inference, + 'selected_inference_mode': selected_inference, + 'config_path': config_paths, + 'result_path': result_paths, + 'matched_baseline_result_path': ';'.join(r['result_path'] for r in baseline_rs), 'status': status, }) return out - def write_csv(path: Path, rows: list[dict[str, Any]]) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open('w', newline='') as f: @@ -277,25 +301,65 @@ def fmt(x: Any) -> str: def write_report(summary_rows: list[dict[str, Any]]) -> None: lines = ['# Integrated TabM final experiment report', ''] - lines += ['Official TabM PLR/PiecewiseLinearEmbeddings baseline configs are preserved exactly; variants add only module flags for RLA, ESAM, MFB, and CF-FISD.', ''] + lines += [ + 'Official TabM PLR/PiecewiseLinearEmbeddings baseline configs are preserved exactly; variants add only module flags for RLA, ESAM, MFB, and CF-FISD.', + 'Deltas and win/loss statuses are computed against `baseline_plr` using the same selected inference mode (`mean`, `best-head`, or `greedy-heads`) as the variant row.', + '', + ] + mean_baseline = { + r['dataset']: r + for r in summary_rows + if r['variant'] == 'baseline_plr' and r['inference_mode'] == 'mean' + } + changed = [] + for r in summary_rows: + if r['variant'] == 'baseline_plr': + continue + base = mean_baseline.get(r['dataset']) + if not base: + continue + old_delta, _ = signed_delta(float(r['mean']), float(base['mean']), r['direction']) + old_status = status_for(old_delta, float(base['std']), int(r['n_seeds']), int(r['n_seeds']) != 3) + if old_status != r['status']: + changed.append((r, old_status, old_delta)) + if changed: + lines += [ + '## Matched-inference correction', + '', + 'The earlier mean-baseline comparison changed when every variant was compared with the matching baseline inference mode. Per the rescue protocol stop condition, no new rescue sweep is claimed in this report.', + '', + '| dataset | variant | inference | mean-baseline status | matched-baseline status | mean-baseline delta | matched delta |', + '|---|---|---|---|---|---:|---:|', + ] + for r, old_status, old_delta in changed: + lines.append( + f"| {r['dataset']} | {r['variant']} | {r['inference_mode']} | {old_status} | {r['status']} | {fmt(old_delta)} | {fmt(r['absolute_delta'])} |" + ) + lines += [''] + lines += ['## Final 3-seed results', ''] - lines += ['| dataset | variant | source config | metric | inference | mean ± std | delta | status |', '|---|---|---|---|---|---:|---:|---|'] + lines += [ + '| dataset | task | variant | selected config | metric | direction | inference | matched baseline mean ± std | result mean ± std | delta | % delta | n | status | config path | result path |', + '|---|---|---|---|---|---|---|---:|---:|---:|---:|---:|---|---|---|', + ] for r in summary_rows: + baseline_mean_std = f"{fmt(r['matched_baseline_mean'])} ± {fmt(r['matched_baseline_std'])}" mean_std = f"{fmt(r['mean'])} ± {fmt(r['std'])}" - lines.append(f"| {r['dataset']} | {r['variant']} | {r['source_variant']} | {r['metric']} | {r['inference_mode']} | {mean_std} | {fmt(r['absolute_delta'])} | {r['status']} |") + lines.append( + f"| {r['dataset']} | {r['task_type']} | {r['variant']} | {r['source_variant']} | {r['metric']} | {r['direction']} | {r['inference_mode']} | {baseline_mean_std} | {mean_std} | {fmt(r['absolute_delta'])} | {fmt(r['percent_delta'])} | {r['n_seeds']} | {r['status']} | `{r['config_path']}` | `{r['result_path']}` |" + ) lines += ['', '## Validation-selected configs', ''] if SELECTED_PATH.exists(): lines += ['| dataset | final variant | selected sweep variant | inference | validation metric |', '|---|---|---|---|---:|'] with SELECTED_PATH.open() as f: for row in csv.DictReader(f): lines.append(f"| {row['dataset']} | {row['final_variant']} | {row['source_variant']} | {row['inference_mode']} | {row['validation_metric']} |") - lines += ['', '## Module wins vs baseline', ''] + lines += ['', '## Module wins vs matched baseline', ''] for variant in FINAL_DISPLAY[1:]: wins = [r['dataset'] for r in summary_rows if r['variant'] == variant and r['status'] in {'clear_win','weak_win'}] lines.append(f"- `{variant}`: {', '.join(wins) if wins else 'none'}") REPORT_PATH.write_text('\n'.join(lines) + '\n') - def stage_wave(manifest: str) -> None: rows = collect_rows() print(f'aggregated complete reports: {len(rows)}') From 3dabcb5b70400e744a6fa0e250d5377b5703458c Mon Sep 17 00:00:00 2001 From: abhipray-chavan <53275745+abhipray-chavan@users.noreply.github.com> Date: Sun, 26 Apr 2026 22:14:09 +0000 Subject: [PATCH 4/7] Improve integrated TabM results --- tabm_integrated/FINAL_EXPERIMENT_REPORT.md | 98 +-- tabm_integrated/paper/bin/run_integrated.py | 25 +- .../paper/exp/final_integrated_audit.csv | 288 +++++-- .../paper/exp/final_integrated_summary.csv | 62 +- .../0/report.json | 227 +++++- .../1/report.json | 229 +++++- .../2/report.json | 229 +++++- .../best_cf_fisd_only-evaluation/0.toml | 109 ++- .../0/report.json | 223 +++++- .../best_cf_fisd_only-evaluation/1.toml | 109 ++- .../1/report.json | 225 +++++- .../best_cf_fisd_only-evaluation/2.toml | 109 ++- .../2/report.json | 225 +++++- .../0/report.json | 244 +++++- .../1/report.json | 240 +++++- .../2/report.json | 241 +++++- .../0/report.json | 522 ++++++++++++- .../1/report.json | 524 ++++++++++++- .../2/report.json | 524 ++++++++++++- .../best_combined-evaluation/0.toml | 130 +++- .../best_combined-evaluation/0/report.json | 518 ++++++++++++- .../best_combined-evaluation/1.toml | 130 +++- .../best_combined-evaluation/1/report.json | 520 ++++++++++++- .../best_combined-evaluation/2.toml | 130 +++- .../best_combined-evaluation/2/report.json | 520 ++++++++++++- .../0/report.json | 529 +++++++++++-- .../1/report.json | 531 ++++++++++++- .../2/report.json | 547 ++++++++++++-- .../0/report.json | 111 ++- .../1/report.json | 105 ++- .../2/report.json | 107 ++- .../best_esam_only-evaluation/0.toml | 6 +- .../best_esam_only-evaluation/0/report.json | 107 ++- .../best_esam_only-evaluation/1.toml | 6 +- .../best_esam_only-evaluation/1/report.json | 103 ++- .../best_esam_only-evaluation/2.toml | 6 +- .../best_esam_only-evaluation/2/report.json | 103 ++- .../0/report.json | 136 +++- .../1/report.json | 119 ++- .../2/report.json | 116 ++- .../0/report.json | 80 +- .../1/report.json | 78 +- .../2/report.json | 80 +- .../best_mfb_only-evaluation/0.toml | 6 +- .../best_mfb_only-evaluation/0/report.json | 76 +- .../best_mfb_only-evaluation/1.toml | 6 +- .../best_mfb_only-evaluation/1/report.json | 74 +- .../best_mfb_only-evaluation/2.toml | 6 +- .../best_mfb_only-evaluation/2/report.json | 76 +- .../0/report.json | 95 ++- .../1/report.json | 87 ++- .../2/report.json | 107 +-- .../0/report.json | 60 +- .../1/report.json | 58 +- .../2/report.json | 62 +- .../best_rla_only-evaluation/0.toml | 6 +- .../best_rla_only-evaluation/0/report.json | 56 +- .../best_rla_only-evaluation/1.toml | 6 +- .../best_rla_only-evaluation/1/report.json | 56 +- .../best_rla_only-evaluation/2.toml | 6 +- .../best_rla_only-evaluation/2/report.json | 58 +- .../0/report.json | 73 +- .../1/report.json | 69 +- .../2/report.json | 78 +- .../manifest_delivery_rescue_combined.txt | 4 + .../manifest_delivery_rescue_final.txt | 15 + .../manifest_delivery_rescue_individual.txt | 62 ++ .../0/DONE | 0 .../0/report.json | 683 +++++++++++++++++ .../deliv_comb_esam_cf-evaluation/0.toml | 161 ++++ .../deliv_comb_esam_cf-evaluation/0/DONE | 0 .../0/report.json | 679 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 706 ++++++++++++++++++ .../0/DONE | 0 .../0/report.json | 633 ++++++++++++++++ .../deliv_comb_rla_cf-evaluation/0.toml | 158 ++++ .../deliv_comb_rla_cf-evaluation/0/DONE | 0 .../0/report.json | 629 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 641 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 240 ++++++ .../deliv_comb_rla_esam-evaluation/0.toml | 50 ++ .../deliv_comb_rla_esam-evaluation/0/DONE | 0 .../0/report.json | 236 ++++++ .../0/DONE | 0 .../0/report.json | 252 +++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../deliv_comb_rla_esam_cf-evaluation/0.toml | 168 +++++ .../deliv_comb_rla_esam_cf-evaluation/0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 706 ++++++++++++++++++ .../0/DONE | 0 .../0/report.json | 622 +++++++++++++++ .../0.toml | 151 ++++ .../0/DONE | 0 .../0/report.json | 618 +++++++++++++++ .../0/DONE | 0 .../0/report.json | 635 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 454 +++++++++++ .../deliv_cf_l0p005_softmax-evaluation/0.toml | 47 ++ .../deliv_cf_l0p005_softmax-evaluation/0/DONE | 0 .../0/report.json | 450 +++++++++++ .../0/DONE | 0 .../0/report.json | 467 ++++++++++++ .../0/DONE | 0 .../0/report.json | 622 +++++++++++++++ .../0.toml | 151 ++++ .../0/DONE | 0 .../0/report.json | 618 +++++++++++++++ .../0/DONE | 0 .../0/report.json | 635 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 454 +++++++++++ .../deliv_cf_l0p01_softmax-evaluation/0.toml | 47 ++ .../deliv_cf_l0p01_softmax-evaluation/0/DONE | 0 .../0/report.json | 450 +++++++++++ .../0/DONE | 0 .../0/report.json | 467 ++++++++++++ .../0/DONE | 0 .../0/report.json | 622 +++++++++++++++ .../0.toml | 151 ++++ .../0/DONE | 0 .../0/report.json | 618 +++++++++++++++ .../0/DONE | 0 .../0/report.json | 635 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 454 +++++++++++ .../deliv_cf_l0p02_softmax-evaluation/0.toml | 47 ++ .../deliv_cf_l0p02_softmax-evaluation/0/DONE | 0 .../0/report.json | 450 +++++++++++ .../0/DONE | 0 .../0/report.json | 467 ++++++++++++ .../0/DONE | 0 .../0/report.json | 229 ++++++ .../0.toml | 43 ++ .../0/DONE | 0 .../0/report.json | 225 ++++++ .../0/DONE | 0 .../0/report.json | 242 ++++++ .../0/DONE | 0 .../0/report.json | 178 +++++ .../0.toml | 43 ++ .../0/DONE | 0 .../0/report.json | 174 +++++ .../0/DONE | 0 .../0/report.json | 191 +++++ .../0/DONE | 0 .../0/report.json | 229 ++++++ .../0.toml | 43 ++ .../0/DONE | 0 .../0/report.json | 225 ++++++ .../0/DONE | 0 .../0/report.json | 242 ++++++ .../0/DONE | 0 .../0/report.json | 178 +++++ .../0.toml | 43 ++ .../0/DONE | 0 .../0/report.json | 174 +++++ .../0/DONE | 0 .../0/report.json | 191 +++++ .../0/DONE | 0 .../0/report.json | 229 ++++++ .../0.toml | 43 ++ .../0/DONE | 0 .../0/report.json | 225 ++++++ .../0/DONE | 0 .../0/report.json | 242 ++++++ .../0/DONE | 0 .../0/report.json | 178 +++++ .../0.toml | 43 ++ .../0/DONE | 0 .../0/report.json | 174 +++++ .../0/DONE | 0 .../0/report.json | 191 +++++ .../0/DONE | 0 .../0/report.json | 229 ++++++ .../0.toml | 43 ++ .../0/DONE | 0 .../0/report.json | 225 ++++++ .../0/DONE | 0 .../0/report.json | 242 ++++++ .../0/DONE | 0 .../0/report.json | 178 +++++ .../0.toml | 43 ++ .../0/DONE | 0 .../0/report.json | 174 +++++ .../0/DONE | 0 .../0/report.json | 191 +++++ .../0/DONE | 0 .../0/report.json | 229 ++++++ .../0.toml | 43 ++ .../0/DONE | 0 .../0/report.json | 225 ++++++ .../0/DONE | 0 .../0/report.json | 252 +++++++ .../0/DONE | 0 .../0/report.json | 178 +++++ .../0.toml | 43 ++ .../0/DONE | 0 .../0/report.json | 174 +++++ .../0/DONE | 0 .../0/report.json | 191 +++++ .../0/DONE | 0 .../0/report.json | 229 ++++++ .../0.toml | 43 ++ .../0/DONE | 0 .../0/report.json | 225 ++++++ .../0/DONE | 0 .../0/report.json | 242 ++++++ .../0/DONE | 0 .../0/report.json | 178 +++++ .../0.toml | 43 ++ .../0/DONE | 0 .../0/report.json | 174 +++++ .../0/DONE | 0 .../0/report.json | 191 +++++ .../0/DONE | 0 .../0/report.json | 212 ++++++ .../0.toml | 50 ++ .../0/DONE | 0 .../0/report.json | 208 ++++++ .../0/DONE | 0 .../0/report.json | 224 ++++++ .../0/DONE | 0 .../0/report.json | 212 ++++++ .../0.toml | 50 ++ .../0/DONE | 0 .../0/report.json | 208 ++++++ .../0/DONE | 0 .../0/report.json | 224 ++++++ .../0/DONE | 0 .../0/report.json | 212 ++++++ .../0.toml | 50 ++ .../0/DONE | 0 .../0/report.json | 208 ++++++ .../0/DONE | 0 .../0/report.json | 224 ++++++ .../0/DONE | 0 .../0/report.json | 212 ++++++ .../0.toml | 50 ++ .../0/DONE | 0 .../0/report.json | 208 ++++++ .../0/DONE | 0 .../0/report.json | 224 ++++++ .../0/DONE | 0 .../0/report.json | 212 ++++++ .../0.toml | 50 ++ .../0/DONE | 0 .../0/report.json | 208 ++++++ .../0/DONE | 0 .../0/report.json | 224 ++++++ .../0/DONE | 0 .../0/report.json | 212 ++++++ .../0.toml | 50 ++ .../0/DONE | 0 .../0/report.json | 208 ++++++ .../0/DONE | 0 .../0/report.json | 224 ++++++ .../0/DONE | 0 .../0/report.json | 212 ++++++ .../0.toml | 50 ++ .../0/DONE | 0 .../0/report.json | 208 ++++++ .../0/DONE | 0 .../0/report.json | 224 ++++++ .../0/DONE | 0 .../0/report.json | 212 ++++++ .../0.toml | 50 ++ .../0/DONE | 0 .../0/report.json | 208 ++++++ .../0/DONE | 0 .../0/report.json | 224 ++++++ .../0/DONE | 0 .../0/report.json | 175 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 171 +++++ .../0/DONE | 0 .../0/report.json | 188 +++++ .../0/DONE | 0 .../0/report.json | 175 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 171 +++++ .../0/DONE | 0 .../0/report.json | 188 +++++ .../0/DONE | 0 .../0/report.json | 175 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 171 +++++ .../0/DONE | 0 .../0/report.json | 188 +++++ .../0/DONE | 0 .../0/report.json | 175 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 171 +++++ .../0/DONE | 0 .../0/report.json | 188 +++++ .../0/DONE | 0 .../0/report.json | 175 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 171 +++++ .../0/DONE | 0 .../0/report.json | 188 +++++ .../0/DONE | 0 .../0/report.json | 175 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 171 +++++ .../0/DONE | 0 .../0/report.json | 188 +++++ .../0/DONE | 0 .../0/report.json | 175 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 171 +++++ .../0/DONE | 0 .../0/report.json | 188 +++++ .../0/DONE | 0 .../0/report.json | 175 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 171 +++++ .../0/DONE | 0 .../0/report.json | 188 +++++ .../0/DONE | 0 .../0/report.json | 175 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 171 +++++ .../0/DONE | 0 .../0/report.json | 188 +++++ .../0/DONE | 0 .../0/report.json | 175 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 171 +++++ .../0/DONE | 0 .../0/report.json | 188 +++++ .../0/DONE | 0 .../0/report.json | 175 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 171 +++++ .../0/DONE | 0 .../0/report.json | 188 +++++ .../0/DONE | 0 .../0/report.json | 175 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 171 +++++ .../0/DONE | 0 .../0/report.json | 188 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 191 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 191 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 194 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 194 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 188 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 191 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 185 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 194 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 190 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 191 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 194 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 194 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 191 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 191 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 194 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 194 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 189 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 191 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 187 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 194 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 190 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 191 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 187 +++++ .../0/DONE | 0 .../0/report.json | 179 +++++ .../0.toml | 40 + .../0/DONE | 0 .../0/report.json | 175 +++++ .../0/DONE | 0 .../0/report.json | 194 +++++ .../paper/exp/rescue_selected_configs.csv | 6 + tabm_integrated/paper/exp/rescue_targets.csv | 6 + .../paper/exp/selected_integrated_configs.csv | 62 +- .../tools/aggregate_integrated_results.py | 26 +- tabm_integrated/tools/delivery_rescue.py | 361 +++++++++ 534 files changed, 61307 insertions(+), 1731 deletions(-) create mode 100644 tabm_integrated/paper/exp/integrated/manifest_delivery_rescue_combined.txt create mode 100644 tabm_integrated/paper/exp/integrated/manifest_delivery_rescue_final.txt create mode 100644 tabm_integrated/paper/exp/integrated/manifest_delivery_rescue_individual.txt create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/rescue_selected_configs.csv create mode 100644 tabm_integrated/paper/exp/rescue_targets.csv create mode 100644 tabm_integrated/tools/delivery_rescue.py diff --git a/tabm_integrated/FINAL_EXPERIMENT_REPORT.md b/tabm_integrated/FINAL_EXPERIMENT_REPORT.md index 82f6fa18f..12144d59b 100644 --- a/tabm_integrated/FINAL_EXPERIMENT_REPORT.md +++ b/tabm_integrated/FINAL_EXPERIMENT_REPORT.md @@ -5,7 +5,7 @@ Deltas and win/loss statuses are computed against `baseline_plr` using the same ## Matched-inference correction -The earlier mean-baseline comparison changed when every variant was compared with the matching baseline inference mode. Per the rescue protocol stop condition, no new rescue sweep is claimed in this report. +The earlier mean-baseline comparison changed when every variant was compared with the matching baseline inference mode. Rescue sweeps are selected against matched validation baselines only. | dataset | variant | inference | mean-baseline status | matched-baseline status | mean-baseline delta | matched delta | |---|---|---|---|---|---:|---:| @@ -22,42 +22,54 @@ The earlier mean-baseline comparison changed when every variant was compared wit | cooking-time | best_mfb_only | greedy-heads | clear_win | weak_win | 0.000407806 | 0.000152541 | | cooking-time | best_cf_fisd_only | greedy-heads | weak_win | loss | 4.32134e-05 | -0.000212051 | | cooking-time | best_combined | greedy-heads | clear_win | weak_win | 0.000354592 | 9.93268e-05 | -| delivery-eta | best_esam_only | greedy-heads | loss | weak_win | -0.000922076 | 2.65273e-05 | +| delivery-eta | best_rla_only | best-head | loss | clear_win | -0.00410785 | 0.00235589 | +| delivery-eta | best_cf_fisd_only | greedy-heads | loss | tie | -0.000948603 | 0 | +| delivery-eta | best_combined | best-head | loss | clear_win | -0.00410785 | 0.00235589 | ## Final 3-seed results -| dataset | task | variant | selected config | metric | direction | inference | matched baseline mean ± std | result mean ± std | delta | % delta | n | status | config path | result path | -|---|---|---|---|---|---|---|---:|---:|---:|---:|---:|---|---|---| -| sberbank-housing | regression | baseline_plr | baseline_plr | RMSE | lower | mean | 0.234418 ± 0.00354822 | 0.234418 ± 0.00354822 | 0 | 0 | 3 | baseline | `exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2` | -| sberbank-housing | regression | best_rla_only | rla_rank2_noise0.001 | RMSE | lower | greedy-heads | 0.234105 ± 0.00278385 | 0.232368 ± 0.00228551 | 0.0017371 | 0.742018 | 3 | weak_win | `exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2` | -| sberbank-housing | regression | best_esam_only | esam_rho0.0025 | RMSE | lower | greedy-heads | 0.234105 ± 0.00278385 | 0.234406 ± 0.00401172 | -0.000301017 | -0.128582 | 3 | loss | `exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2` | -| sberbank-housing | regression | best_mfb_only | mfb_keep0.8 | RMSE | lower | greedy-heads | 0.234105 ± 0.00278385 | 0.2351 ± 0.00395661 | -0.000994573 | -0.424841 | 3 | loss | `exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2` | -| sberbank-housing | regression | best_cf_fisd_only | cf_fisd_only | RMSE | lower | greedy-heads | 0.234105 ± 0.00278385 | 0.233093 ± 0.000585082 | 0.001012 | 0.432283 | 3 | weak_win | `exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2` | -| sberbank-housing | regression | best_combined | mfb_cf_fisd | RMSE | lower | greedy-heads | 0.234105 ± 0.00278385 | 0.234979 ± 0.00337063 | -0.000873715 | -0.373215 | 3 | loss | `exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2` | -| ecom-offers | binclass | baseline_plr | baseline_plr | AUROC | higher | mean | 0.590058 ± 0.000970364 | 0.590058 ± 0.000970364 | 0 | 0 | 3 | baseline | `exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml` | `exp/integrated/final/ecom-offers/baseline_plr-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2` | -| ecom-offers | binclass | best_rla_only | rla_rank2_noise0.0001 | AUROC | higher | best-head | 0.599694 ± 0.00357854 | 0.598447 ± 0.00279735 | -0.00124693 | -0.207927 | 3 | loss | `exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2` | -| ecom-offers | binclass | best_esam_only | esam_rho0.001 | AUROC | higher | greedy-heads | 0.600371 ± 0.00344598 | 0.60042 ± 0.00349335 | 4.90059e-05 | 0.0081626 | 3 | weak_win | `exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2` | -| ecom-offers | binclass | best_mfb_only | mfb_keep0.7 | AUROC | higher | best-head | 0.599694 ± 0.00357854 | 0.609516 ± 0.00208948 | 0.00982263 | 1.63794 | 3 | clear_win | `exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2` | -| ecom-offers | binclass | best_cf_fisd_only | cf_fisd_lambda0.05 | AUROC | higher | greedy-heads | 0.600371 ± 0.00344598 | 0.600205 ± 0.00345825 | -0.000165703 | -0.0276002 | 3 | loss | `exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2` | -| ecom-offers | binclass | best_combined | mfb_cf_fisd | AUROC | higher | greedy-heads | 0.600371 ± 0.00344598 | 0.605211 ± 0.00322248 | 0.00484036 | 0.806228 | 3 | clear_win | `exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2` | -| homesite-insurance | binclass | baseline_plr | baseline_plr | AUROC | higher | mean | 0.962494 ± 0.000178079 | 0.962494 ± 0.000178079 | 0 | 0 | 3 | baseline | `exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2` | -| homesite-insurance | binclass | best_rla_only | rla_rank4_noise1e-05 | AUROC | higher | greedy-heads | 0.962676 ± 0.000276725 | 0.962564 ± 0.000608529 | -0.000111505 | -0.0115828 | 3 | loss | `exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2` | -| homesite-insurance | binclass | best_esam_only | esam_only | AUROC | higher | greedy-heads | 0.962676 ± 0.000276725 | 0.962732 ± 0.00044923 | 5.63183e-05 | 0.00585019 | 3 | weak_win | `exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2` | -| homesite-insurance | binclass | best_mfb_only | mfb_keep0.95 | AUROC | higher | greedy-heads | 0.962676 ± 0.000276725 | 0.962382 ± 0.00045772 | -0.000293222 | -0.030459 | 3 | loss | `exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2` | -| homesite-insurance | binclass | best_cf_fisd_only | cf_fisd_only | AUROC | higher | greedy-heads | 0.962676 ± 0.000276725 | 0.962908 ± 0.000474027 | 0.000232091 | 0.024109 | 3 | weak_win | `exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2` | -| homesite-insurance | binclass | best_combined | mfb_cf_fisd | AUROC | higher | greedy-heads | 0.962676 ± 0.000276725 | 0.962849 ± 0.000437968 | 0.000173469 | 0.0180195 | 3 | weak_win | `exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2` | -| cooking-time | regression | baseline_plr | baseline_plr | RMSE | lower | mean | 0.480381 ± 0.000123385 | 0.480381 ± 0.000123385 | 0 | 0 | 3 | baseline | `exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml` | `exp/integrated/final/cooking-time/baseline_plr-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-evaluation/2` | -| cooking-time | regression | best_rla_only | rla_rank4_noise0.001 | RMSE | lower | greedy-heads | 0.480125 ± 0.000258675 | 0.480146 ± 0.00016089 | -2.08739e-05 | -0.00434759 | 3 | loss | `exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2` | -| cooking-time | regression | best_esam_only | esam_rho0.0025 | RMSE | lower | greedy-heads | 0.480125 ± 0.000258675 | 0.480065 ± 8.02121e-05 | 6.0513e-05 | 0.0126036 | 3 | weak_win | `exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2` | -| cooking-time | regression | best_mfb_only | mfb_keep0.8 | RMSE | lower | greedy-heads | 0.480125 ± 0.000258675 | 0.479973 ± 0.000194552 | 0.000152541 | 0.031771 | 3 | weak_win | `exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2` | -| cooking-time | regression | best_cf_fisd_only | cf_fisd_lambda0.05 | RMSE | lower | greedy-heads | 0.480125 ± 0.000258675 | 0.480337 ± 4.64953e-05 | -0.000212051 | -0.0441658 | 3 | loss | `exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2` | -| cooking-time | regression | best_combined | rla_esam | RMSE | lower | greedy-heads | 0.480125 ± 0.000258675 | 0.480026 ± 9.6554e-05 | 9.93268e-05 | 0.0206877 | 3 | weak_win | `exp/integrated/final/cooking-time/best_combined-evaluation/0.toml;exp/integrated/final/cooking-time/best_combined-evaluation/1.toml;exp/integrated/final/cooking-time/best_combined-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2` | -| delivery-eta | regression | baseline_plr | baseline_plr | RMSE | lower | mean | 0.550226 ± 0.000582292 | 0.550226 ± 0.000582292 | 0 | 0 | 3 | baseline | `exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml` | `exp/integrated/final/delivery-eta/baseline_plr-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2` | -| delivery-eta | regression | best_rla_only | rla_rank2_noise0.001 | RMSE | lower | greedy-heads | 0.551174 ± 0.000706865 | 0.551407 ± 0.00109716 | -0.00023211 | -0.0421119 | 3 | loss | `exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2` | -| delivery-eta | regression | best_esam_only | esam_only | RMSE | lower | greedy-heads | 0.551174 ± 0.000706865 | 0.551148 ± 0.000504719 | 2.65273e-05 | 0.00481288 | 3 | weak_win | `exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2` | -| delivery-eta | regression | best_mfb_only | mfb_keep0.7 | RMSE | lower | greedy-heads | 0.551174 ± 0.000706865 | 0.551366 ± 0.00171387 | -0.000191714 | -0.0347828 | 3 | loss | `exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2` | -| delivery-eta | regression | best_cf_fisd_only | cf_fisd_lambda0.05 | RMSE | lower | greedy-heads | 0.551174 ± 0.000706865 | 0.552434 ± 0.00139117 | -0.00125964 | -0.228537 | 3 | loss | `exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2` | -| delivery-eta | regression | best_combined | rla_esam | RMSE | lower | greedy-heads | 0.551174 ± 0.000706865 | 0.552336 ± 0.000198974 | -0.00116182 | -0.210791 | 3 | loss | `exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2` | +| dataset | task | variant | selected config | metric | direction | inference | matched validation baseline | validation metric | matched baseline test mean ± std | result test mean ± std | delta | % delta | n | status | config path | result path | +|---|---|---|---|---|---|---|---:|---:|---:|---:|---:|---:|---:|---|---|---| +| sberbank-housing | regression | baseline_plr | baseline_plr | RMSE | lower | mean | 0.235525 | 0.235525 | 0.234418 ± 0.00354822 | 0.234418 ± 0.00354822 | 0 | 0 | 3 | baseline | `exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2` | +| sberbank-housing | regression | best_rla_only | rla_rank2_noise0.001 | RMSE | lower | greedy-heads | 0.233757 | 0.233854 | 0.234105 ± 0.00278385 | 0.232368 ± 0.00228551 | 0.0017371 | 0.742018 | 3 | weak_win | `exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2` | +| sberbank-housing | regression | best_esam_only | esam_rho0.0025 | RMSE | lower | greedy-heads | 0.233757 | 0.233815 | 0.234105 ± 0.00278385 | 0.234406 ± 0.00401172 | -0.000301017 | -0.128582 | 3 | loss | `exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2` | +| sberbank-housing | regression | best_mfb_only | mfb_keep0.8 | RMSE | lower | greedy-heads | 0.233757 | 0.233219 | 0.234105 ± 0.00278385 | 0.2351 ± 0.00395661 | -0.000994573 | -0.424841 | 3 | loss | `exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2` | +| sberbank-housing | regression | best_cf_fisd_only | cf_fisd_only | RMSE | lower | greedy-heads | 0.233757 | 0.234079 | 0.234105 ± 0.00278385 | 0.233093 ± 0.000585082 | 0.001012 | 0.432283 | 3 | weak_win | `exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2` | +| sberbank-housing | regression | best_combined | mfb_cf_fisd | RMSE | lower | greedy-heads | 0.233757 | 0.233292 | 0.234105 ± 0.00278385 | 0.234979 ± 0.00337063 | -0.000873715 | -0.373215 | 3 | loss | `exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2` | +| ecom-offers | binclass | baseline_plr | baseline_plr | AUROC | higher | mean | 0.62143 | 0.62143 | 0.590058 ± 0.000970364 | 0.590058 ± 0.000970364 | 0 | 0 | 3 | baseline | `exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml` | `exp/integrated/final/ecom-offers/baseline_plr-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2` | +| ecom-offers | binclass | best_rla_only | rla_rank2_noise0.0001 | AUROC | higher | best-head | 0.635345 | 0.634828 | 0.599694 ± 0.00357854 | 0.598447 ± 0.00279735 | -0.00124693 | -0.207927 | 3 | loss | `exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2` | +| ecom-offers | binclass | best_esam_only | esam_rho0.001 | AUROC | higher | greedy-heads | 0.63659 | 0.636656 | 0.600371 ± 0.00344598 | 0.60042 ± 0.00349335 | 4.90059e-05 | 0.0081626 | 3 | weak_win | `exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2` | +| ecom-offers | binclass | best_mfb_only | mfb_keep0.7 | AUROC | higher | best-head | 0.635345 | 0.653977 | 0.599694 ± 0.00357854 | 0.609516 ± 0.00208948 | 0.00982263 | 1.63794 | 3 | clear_win | `exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2` | +| ecom-offers | binclass | best_cf_fisd_only | cf_fisd_lambda0.05 | AUROC | higher | greedy-heads | 0.63659 | 0.63644 | 0.600371 ± 0.00344598 | 0.600205 ± 0.00345825 | -0.000165703 | -0.0276002 | 3 | loss | `exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2` | +| ecom-offers | binclass | best_combined | mfb_cf_fisd | AUROC | higher | greedy-heads | 0.63659 | 0.64852 | 0.600371 ± 0.00344598 | 0.605211 ± 0.00322248 | 0.00484036 | 0.806228 | 3 | clear_win | `exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2` | +| homesite-insurance | binclass | baseline_plr | baseline_plr | AUROC | higher | mean | 0.95842 | 0.95842 | 0.962494 ± 0.000178079 | 0.962494 ± 0.000178079 | 0 | 0 | 3 | baseline | `exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2` | +| homesite-insurance | binclass | best_rla_only | rla_rank4_noise1e-05 | AUROC | higher | greedy-heads | 0.95914 | 0.958978 | 0.962676 ± 0.000276725 | 0.962564 ± 0.000608529 | -0.000111505 | -0.0115828 | 3 | loss | `exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2` | +| homesite-insurance | binclass | best_esam_only | esam_only | AUROC | higher | greedy-heads | 0.95914 | 0.959081 | 0.962676 ± 0.000276725 | 0.962732 ± 0.00044923 | 5.63183e-05 | 0.00585019 | 3 | weak_win | `exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2` | +| homesite-insurance | binclass | best_mfb_only | mfb_keep0.95 | AUROC | higher | greedy-heads | 0.95914 | 0.959182 | 0.962676 ± 0.000276725 | 0.962382 ± 0.00045772 | -0.000293222 | -0.030459 | 3 | loss | `exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2` | +| homesite-insurance | binclass | best_cf_fisd_only | cf_fisd_only | AUROC | higher | greedy-heads | 0.95914 | 0.959528 | 0.962676 ± 0.000276725 | 0.962908 ± 0.000474027 | 0.000232091 | 0.024109 | 3 | weak_win | `exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2` | +| homesite-insurance | binclass | best_combined | mfb_cf_fisd | AUROC | higher | greedy-heads | 0.95914 | 0.959633 | 0.962676 ± 0.000276725 | 0.962849 ± 0.000437968 | 0.000173469 | 0.0180195 | 3 | weak_win | `exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml` | `exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2` | +| cooking-time | regression | baseline_plr | baseline_plr | RMSE | lower | mean | 0.462709 | 0.462709 | 0.480381 ± 0.000123385 | 0.480381 ± 0.000123385 | 0 | 0 | 3 | baseline | `exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml` | `exp/integrated/final/cooking-time/baseline_plr-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-evaluation/2` | +| cooking-time | regression | best_rla_only | rla_rank4_noise0.001 | RMSE | lower | greedy-heads | 0.46247 | 0.462442 | 0.480125 ± 0.000258675 | 0.480146 ± 0.00016089 | -2.08739e-05 | -0.00434759 | 3 | loss | `exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2` | +| cooking-time | regression | best_esam_only | esam_rho0.0025 | RMSE | lower | greedy-heads | 0.46247 | 0.462478 | 0.480125 ± 0.000258675 | 0.480065 ± 8.02121e-05 | 6.0513e-05 | 0.0126036 | 3 | weak_win | `exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2` | +| cooking-time | regression | best_mfb_only | mfb_keep0.8 | RMSE | lower | greedy-heads | 0.46247 | 0.462503 | 0.480125 ± 0.000258675 | 0.479973 ± 0.000194552 | 0.000152541 | 0.031771 | 3 | weak_win | `exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2` | +| cooking-time | regression | best_cf_fisd_only | cf_fisd_lambda0.05 | RMSE | lower | greedy-heads | 0.46247 | 0.462954 | 0.480125 ± 0.000258675 | 0.480337 ± 4.64953e-05 | -0.000212051 | -0.0441658 | 3 | loss | `exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2` | +| cooking-time | regression | best_combined | rla_esam | RMSE | lower | greedy-heads | 0.46247 | 0.462443 | 0.480125 ± 0.000258675 | 0.480026 ± 9.6554e-05 | 9.93268e-05 | 0.0206877 | 3 | weak_win | `exp/integrated/final/cooking-time/best_combined-evaluation/0.toml;exp/integrated/final/cooking-time/best_combined-evaluation/1.toml;exp/integrated/final/cooking-time/best_combined-evaluation/2.toml` | `exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2` | +| delivery-eta | regression | baseline_plr | baseline_plr | RMSE | lower | mean | 0.55595 | 0.55595 | 0.550226 ± 0.000582292 | 0.550226 ± 0.000582292 | 0 | 0 | 3 | baseline | `exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml` | `exp/integrated/final/delivery-eta/baseline_plr-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2` | +| delivery-eta | regression | best_rla_only | deliv_rla_r4_n0p0001_lr0p5_fr0 | RMSE | lower | best-head | 0.557964 | 0.557868 | 0.55669 ± 0.000859735 | 0.554334 ± 0.00266191 | 0.00235589 | 0.423196 | 3 | clear_win | `exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0;exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1;exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2` | +| delivery-eta | regression | best_esam_only | deliv_esam_rho0p001_sf0p25_adapterf | RMSE | lower | mean | 0.55595 | 0.555934 | 0.550226 ± 0.000582292 | 0.551263 ± 0.00129165 | -0.00103689 | -0.188448 | 3 | loss | `exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_esam_only-evaluation/0;exp/integrated/final/delivery-eta/best_esam_only-evaluation/1;exp/integrated/final/delivery-eta/best_esam_only-evaluation/2` | +| delivery-eta | regression | best_mfb_only | deliv_mfb_k0p975_sf0p5_numerical_only | RMSE | lower | mean | 0.55595 | 0.556002 | 0.550226 ± 0.000582292 | 0.5502 ± 0.000181893 | 2.62743e-05 | 0.00477518 | 3 | weak_win | `exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2` | +| delivery-eta | regression | best_cf_fisd_only | deliv_cf_l0p005_consensus_raw | RMSE | lower | greedy-heads | 0.555387 | 0.555387 | 0.551174 ± 0.000706865 | 0.551174 ± 0.000706865 | 0 | 0 | 3 | tie | `exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2` | +| delivery-eta | regression | best_combined | deliv_comb_rla_cf | RMSE | lower | best-head | 0.557964 | 0.557868 | 0.55669 ± 0.000859735 | 0.554334 ± 0.00266191 | 0.00235589 | 0.423196 | 3 | clear_win | `exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml` | `exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0;exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1;exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2` | + +## Rescue validation selections + +| dataset | final variant | selected rescue config | inference | validation metric | matched validation baseline | validation delta | validation status | confirmed | +|---|---|---|---|---:|---:|---:|---|---| +| delivery-eta | best_rla_only | deliv_rla_r4_n0p0001_lr0p5_fr0 | best-head | 0.5572652691682626 | 0.5583881453279098 | 0.0011228761596472348 | validation_win | True | +| delivery-eta | best_esam_only | deliv_esam_rho0p001_sf0p25_adapterf | mean | 0.5558269510040955 | 0.5558749906446844 | 4.8039640588859456e-05 | validation_win | True | +| delivery-eta | best_mfb_only | deliv_mfb_k0p975_sf0p5_numerical_only | mean | 0.5558048331442785 | 0.5558749906446844 | 7.015750040584301e-05 | validation_win | True | +| delivery-eta | best_cf_fisd_only | deliv_cf_l0p005_consensus_raw | greedy-heads | 0.5553348954063525 | 0.5553348954063525 | 0.0 | close_no_validation_win | True | +| delivery-eta | best_combined | deliv_comb_rla_cf | best-head | 0.5572652691682626 | 0.5583881453279098 | 0.0011228761596472348 | validation_win | True | ## Validation-selected configs @@ -88,16 +100,16 @@ The earlier mean-baseline comparison changed when every variant was compared wit | cooking-time | best_cf_fisd_only | cf_fisd_lambda0.05 | greedy-heads | 0.4628482268025946 | | cooking-time | best_combined | rla_esam | greedy-heads | 0.4624234187671886 | | delivery-eta | baseline_plr | baseline_plr | mean | | -| delivery-eta | best_rla_only | rla_rank2_noise0.001 | greedy-heads | 0.5552720495549479 | -| delivery-eta | best_esam_only | esam_only | greedy-heads | 0.5551513290724005 | -| delivery-eta | best_mfb_only | mfb_keep0.7 | greedy-heads | 0.5551008108444078 | -| delivery-eta | best_cf_fisd_only | cf_fisd_lambda0.05 | greedy-heads | 0.5556691040453957 | -| delivery-eta | best_combined | rla_esam | greedy-heads | 0.5552988041877751 | +| delivery-eta | best_rla_only | deliv_rla_r4_n0p0001_lr0p5_fr0 | best-head | 0.5572652691682626 | +| delivery-eta | best_esam_only | deliv_esam_rho0p001_sf0p25_adapterf | mean | 0.5558269510040955 | +| delivery-eta | best_mfb_only | deliv_mfb_k0p975_sf0p5_numerical_only | mean | 0.5558048331442785 | +| delivery-eta | best_cf_fisd_only | deliv_cf_l0p005_consensus_raw | greedy-heads | 0.5553348954063525 | +| delivery-eta | best_combined | deliv_comb_rla_cf | best-head | 0.5572652691682626 | ## Module wins vs matched baseline -- `best_rla_only`: sberbank-housing -- `best_esam_only`: ecom-offers, homesite-insurance, cooking-time, delivery-eta -- `best_mfb_only`: ecom-offers, cooking-time +- `best_rla_only`: sberbank-housing, delivery-eta +- `best_esam_only`: ecom-offers, homesite-insurance, cooking-time +- `best_mfb_only`: ecom-offers, cooking-time, delivery-eta - `best_cf_fisd_only`: sberbank-housing, homesite-insurance -- `best_combined`: ecom-offers, homesite-insurance, cooking-time +- `best_combined`: ecom-offers, homesite-insurance, cooking-time, delivery-eta diff --git a/tabm_integrated/paper/bin/run_integrated.py b/tabm_integrated/paper/bin/run_integrated.py index 6dabe48d5..006f440ed 100644 --- a/tabm_integrated/paper/bin/run_integrated.py +++ b/tabm_integrated/paper/bin/run_integrated.py @@ -514,19 +514,31 @@ def __init__( self.mfb_use_soft_mask = bool(self.mfb_cfg.get('use_soft_mask', False)) self.mfb_mask_strength = float(self.mfb_cfg.get('mask_strength', 1.0)) self.mfb_warmup_epochs = int(self.mfb_cfg.get('warmup_epochs', 0)) + self.mfb_start_epoch = int(self.mfb_cfg.get('start_epoch', 0)) + self.mfb_group_mode = str(self.mfb_cfg.get('group_mode', 'feature_group')) + self.mfb_categorical_handling = str(self.mfb_cfg.get('categorical_handling', 'drop_allowed')) self.mfb_epoch = 0 self.mfb_feature_widths = [int(x) for x in first_adapter_sections] if self.mfb_enabled: assert self.k is not None if self.mfb_mask_granularity != 'feature_group': raise ValueError(f'Unsupported MFB mask_granularity={self.mfb_mask_granularity!r}') + if self.mfb_group_mode not in {'feature_group', 'numerical_only', 'per_member'}: + raise ValueError(f'Unsupported MFB group_mode={self.mfb_group_mode!r}') + protected_feature_ids = list(self.mfb_cfg.get('protected_feature_ids') or []) + if ( + self.mfb_group_mode == 'numerical_only' + or self.mfb_categorical_handling in {'no_cat_drop', 'num_only'} + ): + protected_feature_ids.extend(range(n_num_features, n_num_features + len(cat_cardinalities))) + protected_feature_ids = sorted(set(int(x) for x in protected_feature_ids)) feature_mask, dim_mask, mask_stats = _make_mfb_feature_group_mask( k=self.k, feature_widths=self.mfb_feature_widths, keep_rate=self.mfb_keep_rate, seed=int(self.mfb_cfg.get('mask_seed', 0)), anchor_fraction=float(self.mfb_cfg.get('anchor_fraction', 0.0)), - protected_feature_ids=self.mfb_cfg.get('protected_feature_ids'), + protected_feature_ids=protected_feature_ids, ) self.register_buffer('mfb_fixed_feature_mask', torch.from_numpy(feature_mask), persistent=True) self.register_buffer('mfb_fixed_dim_mask', torch.from_numpy(dim_mask), persistent=True) @@ -544,7 +556,8 @@ def _current_mfb_mask_strength(self) -> float: return 1.0 if self.mfb_warmup_epochs <= 0: return self.mfb_mask_strength - return self.mfb_mask_strength * min(1.0, max(0.0, float(self.mfb_epoch) / float(self.mfb_warmup_epochs))) + effective_epoch = max(0, self.mfb_epoch - self.mfb_start_epoch) + return self.mfb_mask_strength * min(1.0, max(0.0, float(effective_epoch) / float(self.mfb_warmup_epochs))) def _sample_mfb_mask(self, device_: torch.device, dtype: torch.dtype) -> Tensor: assert self.k is not None @@ -565,6 +578,8 @@ def _sample_mfb_mask(self, device_: torch.device, dtype: torch.dtype) -> Tensor: def _apply_mfb_mask(self, x: Tensor) -> Tensor: if not self.mfb_enabled or self.mfb_mask_mode == 'none': return x + if self.mfb_epoch < self.mfb_start_epoch: + return x if self.mfb_mask_mode == 'member_fixed': raw_mask = self.mfb_fixed_dim_mask.to(device=x.device, dtype=x.dtype) elif self.mfb_mask_mode == 'stochastic': @@ -814,6 +829,7 @@ def main( ) cf_fisd_lambda = float(cf_fisd_cfg.get('lambda', 0.0)) cf_fisd_variant = str(cf_fisd_cfg.get('variant', 'raw')) + cf_fisd_start_epoch = int(cf_fisd_cfg.get('start_epoch', 0)) cf_fisd_r1_param = _get_first_adapter_for_cf_fisd(root_model.backbone) report['cf_fisd'] = { 'lambda': cf_fisd_lambda, @@ -823,6 +839,8 @@ def main( 'd_features': list(cf_fisd_d_features), 'n_features': cf_fisd_n_features, 'teacher_dir': str(cf_fisd_cfg['teacher_dir']), + 'start_epoch': cf_fisd_start_epoch, + 'mode': cf_fisd_cfg.get('mode', cf_fisd_variant), } else: cf_fisd_lambda = 0.0 @@ -830,12 +848,15 @@ def main( cf_fisd_d_features = [] cf_fisd_teachers = {} cf_fisd_member_groups = {} + cf_fisd_start_epoch = 0 cf_fisd_r1_param = None report['cf_fisd'] = {'lambda': 0.0} def compute_cf_fisd_penalty() -> Tensor: if cf_fisd_r1_param is None or cf_fisd_lambda <= 0.0: return Y_train.new_zeros((), dtype=torch.float32) + if step // epoch_size < cf_fisd_start_epoch: + return Y_train.new_zeros((), dtype=torch.float32) return lib.cf_fisd.cf_fisd_loss( cf_fisd_r1_param, cf_fisd_teachers, diff --git a/tabm_integrated/paper/exp/final_integrated_audit.csv b/tabm_integrated/paper/exp/final_integrated_audit.csv index 00cd114aa..eacfa652c 100644 --- a/tabm_integrated/paper/exp/final_integrated_audit.csv +++ b/tabm_integrated/paper/exp/final_integrated_audit.csv @@ -788,54 +788,252 @@ final,cooking-time,best_combined,2,RMSE,rmse,lower,0.4623945128795896,-0.4623945 final,delivery-eta,baseline_plr,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0,False,2184 final,delivery-eta,baseline_plr,1,RMSE,rmse,lower,0.5562247345576334,-0.5562247345576334,0.550284872190118,mean,exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml,exp/integrated/final/delivery-eta/baseline_plr-evaluation/1,False,1638 final,delivery-eta,baseline_plr,2,RMSE,rmse,lower,0.5557493871736087,-0.5557493871736087,0.5507763801521746,mean,exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml,exp/integrated/final/delivery-eta/baseline_plr-evaluation/2,False,1638 -final,delivery-eta,best_rla_only,0,RMSE,rmse,lower,0.5556336245387727,-0.5556336245387727,0.5518591323593222,mean,exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_rla_only-evaluation/0,False,1365 -final,delivery-eta,best_rla_only,1,RMSE,rmse,lower,0.5562834011722252,-0.5562834011722252,0.5505008118837418,mean,exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_rla_only-evaluation/1,False,1365 -final,delivery-eta,best_rla_only,2,RMSE,rmse,lower,0.5564119371157284,-0.5564119371157284,0.5491394300416609,mean,exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_rla_only-evaluation/2,False,1365 -final,delivery-eta,best_esam_only,0,RMSE,rmse,lower,0.5556360918188752,-0.5556360918188752,0.5494649592968895,mean,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0,False,2184 -final,delivery-eta,best_esam_only,1,RMSE,rmse,lower,0.5559996009180297,-0.5559996009180297,0.5517844674752832,mean,exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_esam_only-evaluation/1,False,1638 -final,delivery-eta,best_esam_only,2,RMSE,rmse,lower,0.555909194911898,-0.555909194911898,0.5492662194956084,mean,exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_esam_only-evaluation/2,False,1638 -final,delivery-eta,best_mfb_only,0,RMSE,rmse,lower,0.5576335149669441,-0.5576335149669441,0.5507854163911843,mean,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0,False,2730 -final,delivery-eta,best_mfb_only,1,RMSE,rmse,lower,0.5557639999222178,-0.5557639999222178,0.5537457867240355,mean,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1,False,1638 -final,delivery-eta,best_mfb_only,2,RMSE,rmse,lower,0.556207481645197,-0.556207481645197,0.551272504610975,mean,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2,False,3003 -final,delivery-eta,best_cf_fisd_only,0,RMSE,rmse,lower,0.5562445854609275,-0.5562445854609275,0.5504166498793033,mean,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0,False,2184 -final,delivery-eta,best_cf_fisd_only,1,RMSE,rmse,lower,0.5564364677950075,-0.5564364677950075,0.550973168882935,mean,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1,False,1365 -final,delivery-eta,best_cf_fisd_only,2,RMSE,rmse,lower,0.5569470014167281,-0.5569470014167281,0.5474489469980929,mean,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2,False,1365 -final,delivery-eta,best_combined,0,RMSE,rmse,lower,0.5559417888696784,-0.5559417888696784,0.5541331785404355,mean,exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml,exp/integrated/final/delivery-eta/best_combined-evaluation/0,False,1365 -final,delivery-eta,best_combined,1,RMSE,rmse,lower,0.5563776030607058,-0.5563776030607058,0.5504132928776152,mean,exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml,exp/integrated/final/delivery-eta/best_combined-evaluation/1,False,1638 -final,delivery-eta,best_combined,2,RMSE,rmse,lower,0.556831808887752,-0.556831808887752,0.5516551774702309,mean,exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml,exp/integrated/final/delivery-eta/best_combined-evaluation/2,False,819 +final,delivery-eta,best_rla_only,0,RMSE,rmse,lower,0.5559356776481553,-0.5559356776481553,0.55135486033622,mean,exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_rla_only-evaluation/0,False,1365 +final,delivery-eta,best_rla_only,1,RMSE,rmse,lower,0.5564338969448529,-0.5564338969448529,0.5491088203688107,mean,exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_rla_only-evaluation/1,False,1365 +final,delivery-eta,best_rla_only,2,RMSE,rmse,lower,0.5562272527951271,-0.5562272527951271,0.5494697322909359,mean,exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_rla_only-evaluation/2,False,1911 +final,delivery-eta,best_esam_only,0,RMSE,rmse,lower,0.5558269510040955,-0.5558269510040955,0.5527269612276788,mean,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0,False,2730 +final,delivery-eta,best_esam_only,1,RMSE,rmse,lower,0.5562247345576334,-0.5562247345576334,0.550284872190118,mean,exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_esam_only-evaluation/1,False,1638 +final,delivery-eta,best_esam_only,2,RMSE,rmse,lower,0.5557493871736087,-0.5557493871736087,0.5507763801521746,mean,exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_esam_only-evaluation/2,False,1638 +final,delivery-eta,best_mfb_only,1,RMSE,rmse,lower,0.5559520276955289,-0.5559520276955289,0.5503973468399834,mean,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1,False,1638 +final,delivery-eta,best_mfb_only,2,RMSE,rmse,lower,0.5562502378827562,-0.5562502378827562,0.5500394546922427,mean,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2,False,1638 +final,delivery-eta,best_mfb_only,0,RMSE,rmse,lower,0.5558048331442785,-0.5558048331442785,0.5501619198178428,mean,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0,False,2184 +final,delivery-eta,best_cf_fisd_only,1,RMSE,rmse,lower,0.5562247345576334,-0.5562247345576334,0.550284872190118,mean,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1,False,1638 +final,delivery-eta,best_cf_fisd_only,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0,False,2184 +final,delivery-eta,best_cf_fisd_only,2,RMSE,rmse,lower,0.5557493871736087,-0.5557493871736087,0.5507763801521746,mean,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2,False,1638 +final,delivery-eta,best_combined,1,RMSE,rmse,lower,0.5564338969448529,-0.5564338969448529,0.5491088203688107,mean,exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml,exp/integrated/final/delivery-eta/best_combined-evaluation/1,False,1365 +final,delivery-eta,best_combined,0,RMSE,rmse,lower,0.5559356776481553,-0.5559356776481553,0.55135486033622,mean,exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml,exp/integrated/final/delivery-eta/best_combined-evaluation/0,False,1365 +final,delivery-eta,best_combined,2,RMSE,rmse,lower,0.5562272527951271,-0.5562272527951271,0.5494697322909359,mean,exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml,exp/integrated/final/delivery-eta/best_combined-evaluation/2,False,1911 final,delivery-eta,baseline_plr,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml,exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/0,False,2184 final,delivery-eta,baseline_plr,1,RMSE,rmse,lower,0.5578575885405295,-0.5578575885405295,0.5568376961917461,best-head,exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml,exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/1,False,1638 final,delivery-eta,baseline_plr,2,RMSE,rmse,lower,0.5576450053690213,-0.5576450053690213,0.5574656479348947,best-head,exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml,exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/2,False,1638 final,delivery-eta,baseline_plr,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml,exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0,False,2184 final,delivery-eta,baseline_plr,1,RMSE,rmse,lower,0.5553836483690917,-0.5553836483690917,0.5515298017800222,greedy-heads,exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml,exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1,False,1638 final,delivery-eta,baseline_plr,2,RMSE,rmse,lower,0.5554437183690072,-0.5554437183690072,0.5516331354574737,greedy-heads,exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml,exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2,False,1638 -final,delivery-eta,best_rla_only,0,RMSE,rmse,lower,0.5574634025946379,-0.5574634025946379,0.5547066537478277,best-head,exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0,False,1365 -final,delivery-eta,best_rla_only,1,RMSE,rmse,lower,0.5576988200546102,-0.5576988200546102,0.551726996987587,best-head,exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1,False,1365 -final,delivery-eta,best_rla_only,2,RMSE,rmse,lower,0.5580165521873937,-0.5580165521873937,0.5499792279309027,best-head,exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2,False,1365 -final,delivery-eta,best_rla_only,0,RMSE,rmse,lower,0.5552720495549479,-0.5552720495549479,0.5524877260752137,greedy-heads,exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0,False,1365 -final,delivery-eta,best_rla_only,1,RMSE,rmse,lower,0.55584140086384,-0.55584140086384,0.5514378793216401,greedy-heads,exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1,False,1365 -final,delivery-eta,best_rla_only,2,RMSE,rmse,lower,0.555843679564389,-0.555843679564389,0.5502940789713319,greedy-heads,exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2,False,1365 -final,delivery-eta,best_esam_only,0,RMSE,rmse,lower,0.5582863288518123,-0.5582863288518123,0.5560390501031047,best-head,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/0,False,2184 -final,delivery-eta,best_esam_only,1,RMSE,rmse,lower,0.5579506166150647,-0.5579506166150647,0.5600144253303337,best-head,exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/1,False,1638 -final,delivery-eta,best_esam_only,2,RMSE,rmse,lower,0.5578059530007539,-0.5578059530007539,0.5592548752846349,best-head,exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/2,False,1638 -final,delivery-eta,best_esam_only,0,RMSE,rmse,lower,0.5551513290724005,-0.5551513290724005,0.5507488105906536,greedy-heads,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0,False,2184 -final,delivery-eta,best_esam_only,1,RMSE,rmse,lower,0.5557057343261759,-0.5557057343261759,0.5517152752978882,greedy-heads,exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1,False,1638 -final,delivery-eta,best_esam_only,2,RMSE,rmse,lower,0.5554111756694609,-0.5554111756694609,0.5509796867295551,greedy-heads,exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2,False,1638 -final,delivery-eta,best_mfb_only,0,RMSE,rmse,lower,0.5595885535124743,-0.5595885535124743,0.5534224254724212,best-head,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/0,False,2730 -final,delivery-eta,best_mfb_only,1,RMSE,rmse,lower,0.5585395946793884,-0.5585395946793884,0.5622948696369855,best-head,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/1,False,1638 -final,delivery-eta,best_mfb_only,2,RMSE,rmse,lower,0.5610582099606907,-0.5610582099606907,0.5524986222625401,best-head,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/2,False,3003 -final,delivery-eta,best_mfb_only,0,RMSE,rmse,lower,0.5551008108444078,-0.5551008108444078,0.5501919833980724,greedy-heads,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0,False,2730 -final,delivery-eta,best_mfb_only,1,RMSE,rmse,lower,0.5549583042651809,-0.5549583042651809,0.5533328640985606,greedy-heads,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1,False,1638 -final,delivery-eta,best_mfb_only,2,RMSE,rmse,lower,0.55517701590878,-0.55517701590878,0.5505736480564649,greedy-heads,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2,False,3003 -final,delivery-eta,best_cf_fisd_only,0,RMSE,rmse,lower,0.557661572508718,-0.557661572508718,0.553144673358601,best-head,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/0,False,2184 -final,delivery-eta,best_cf_fisd_only,1,RMSE,rmse,lower,0.5570080264093609,-0.5570080264093609,0.5540194450901057,best-head,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/1,False,1365 -final,delivery-eta,best_cf_fisd_only,2,RMSE,rmse,lower,0.5578165315968546,-0.5578165315968546,0.5527351028922852,best-head,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/2,False,1365 -final,delivery-eta,best_cf_fisd_only,0,RMSE,rmse,lower,0.5556691040453957,-0.5556691040453957,0.5521822756750591,greedy-heads,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0,False,2184 -final,delivery-eta,best_cf_fisd_only,1,RMSE,rmse,lower,0.5559175580175352,-0.5559175580175352,0.5539339615490284,greedy-heads,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1,False,1365 -final,delivery-eta,best_cf_fisd_only,2,RMSE,rmse,lower,0.5559178796729317,-0.5559178796729317,0.551186027318127,greedy-heads,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2,False,1365 -final,delivery-eta,best_combined,0,RMSE,rmse,lower,0.5572907783524989,-0.5572907783524989,0.5545303498210034,best-head,exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml,exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0,False,1365 -final,delivery-eta,best_combined,2,RMSE,rmse,lower,0.5577944659141506,-0.5577944659141506,0.5536175199956386,best-head,exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml,exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2,False,819 -final,delivery-eta,best_combined,1,RMSE,rmse,lower,0.5571610811723605,-0.5571610811723605,0.5529389828325415,best-head,exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml,exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1,False,1638 -final,delivery-eta,best_combined,0,RMSE,rmse,lower,0.5552988041877751,-0.5552988041877751,0.5525187688548042,greedy-heads,exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml,exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0,False,1365 -final,delivery-eta,best_combined,2,RMSE,rmse,lower,0.5564955940706238,-0.5564955940706238,0.5523659114897095,greedy-heads,exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml,exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2,False,819 -final,delivery-eta,best_combined,1,RMSE,rmse,lower,0.5558474595058753,-0.5558474595058753,0.5521241448870569,greedy-heads,exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml,exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1,False,1638 +final,delivery-eta,best_rla_only,0,RMSE,rmse,lower,0.5572652691682626,-0.5572652691682626,0.5524207799310179,best-head,exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0,False,1365 +final,delivery-eta,best_rla_only,1,RMSE,rmse,lower,0.5577909128836495,-0.5577909128836495,0.5532065757037241,best-head,exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1,False,1365 +final,delivery-eta,best_rla_only,2,RMSE,rmse,lower,0.5585470113340857,-0.5585470113340857,0.5573737419749005,best-head,exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2,False,1911 +final,delivery-eta,best_rla_only,0,RMSE,rmse,lower,0.5553930389270275,-0.5553930389270275,0.5519483119271499,greedy-heads,exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0,False,1365 +final,delivery-eta,best_rla_only,1,RMSE,rmse,lower,0.5557605143473513,-0.5557605143473513,0.5514412571081766,greedy-heads,exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1,False,1365 +final,delivery-eta,best_rla_only,2,RMSE,rmse,lower,0.5554844141815782,-0.5554844141815782,0.5520847398512095,greedy-heads,exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2,False,1911 +final,delivery-eta,best_esam_only,2,RMSE,rmse,lower,0.5576450053690213,-0.5576450053690213,0.5574656479348947,best-head,exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/2,False,1638 +final,delivery-eta,best_esam_only,1,RMSE,rmse,lower,0.5578575885405295,-0.5578575885405295,0.5568376961917461,best-head,exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/1,False,1638 +final,delivery-eta,best_esam_only,0,RMSE,rmse,lower,0.5596782586864594,-0.5596782586864594,0.5565908577070521,best-head,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/0,False,2730 +final,delivery-eta,best_esam_only,2,RMSE,rmse,lower,0.5554437183690072,-0.5554437183690072,0.5516331354574737,greedy-heads,exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2,False,1638 +final,delivery-eta,best_esam_only,1,RMSE,rmse,lower,0.5553836483690917,-0.5553836483690917,0.5515298017800222,greedy-heads,exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1,False,1638 +final,delivery-eta,best_esam_only,0,RMSE,rmse,lower,0.555452544543325,-0.555452544543325,0.5532771704027001,greedy-heads,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0,False,2730 +final,delivery-eta,best_mfb_only,1,RMSE,rmse,lower,0.5576769100347859,-0.5576769100347859,0.5558933529149844,best-head,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/1,False,1638 +final,delivery-eta,best_mfb_only,2,RMSE,rmse,lower,0.558478390179117,-0.558478390179117,0.5526173066624255,best-head,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/2,False,1638 +final,delivery-eta,best_mfb_only,0,RMSE,rmse,lower,0.5590051856389011,-0.5590051856389011,0.5534488118080276,best-head,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/0,False,2184 +final,delivery-eta,best_mfb_only,1,RMSE,rmse,lower,0.55527543085353,-0.55527543085353,0.5524079939671243,greedy-heads,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1,False,1638 +final,delivery-eta,best_mfb_only,2,RMSE,rmse,lower,0.5553606274153348,-0.5553606274153348,0.5524157087402433,greedy-heads,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2,False,1638 +final,delivery-eta,best_mfb_only,0,RMSE,rmse,lower,0.5553099940581016,-0.5553099940581016,0.5510631408064661,greedy-heads,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0,False,2184 +final,delivery-eta,best_cf_fisd_only,2,RMSE,rmse,lower,0.5576450053690213,-0.5576450053690213,0.5574656479348947,best-head,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/2,False,1638 +final,delivery-eta,best_cf_fisd_only,1,RMSE,rmse,lower,0.5578575885405295,-0.5578575885405295,0.5568376961917461,best-head,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/1,False,1638 +final,delivery-eta,best_cf_fisd_only,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/0,False,2184 +final,delivery-eta,best_cf_fisd_only,2,RMSE,rmse,lower,0.5554437183690072,-0.5554437183690072,0.5516331354574737,greedy-heads,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2,False,1638 +final,delivery-eta,best_cf_fisd_only,1,RMSE,rmse,lower,0.5553836483690917,-0.5553836483690917,0.5515298017800222,greedy-heads,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1,False,1638 +final,delivery-eta,best_cf_fisd_only,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0,False,2184 +final,delivery-eta,best_combined,1,RMSE,rmse,lower,0.5577909128836495,-0.5577909128836495,0.5532065757037241,best-head,exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml,exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1,False,1365 +final,delivery-eta,best_combined,0,RMSE,rmse,lower,0.5572652691682626,-0.5572652691682626,0.5524207799310179,best-head,exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml,exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0,False,1365 +final,delivery-eta,best_combined,2,RMSE,rmse,lower,0.5585470113340857,-0.5585470113340857,0.5573737419749005,best-head,exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml,exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2,False,1911 +final,delivery-eta,best_combined,1,RMSE,rmse,lower,0.5557605143473513,-0.5557605143473513,0.5514412571081766,greedy-heads,exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml,exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1,False,1365 +final,delivery-eta,best_combined,0,RMSE,rmse,lower,0.5553930389270275,-0.5553930389270275,0.5519483119271499,greedy-heads,exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml,exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0,False,1365 +final,delivery-eta,best_combined,2,RMSE,rmse,lower,0.5554844141815782,-0.5554844141815782,0.5520847398512095,greedy-heads,exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml,exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2,False,1911 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p25_fr0,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p5_fr0,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p25_fr0,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p5_fr0,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p25_fr0,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p5_fr0,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p25_fr0,0,RMSE,rmse,lower,0.5558022325623276,-0.5558022325623276,0.5512480144508224,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5558022325623276,-0.5558022325623276,0.5512480144508224,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p5_fr0,0,RMSE,rmse,lower,0.5558360123700841,-0.5558360123700841,0.5491424149361828,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5558360123700841,-0.5558360123700841,0.5491424149361828,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p25_fr0,0,RMSE,rmse,lower,0.5560690371152802,-0.5560690371152802,0.5483022050176959,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5558022325623276,-0.5558022325623276,0.5512480144508224,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p5_fr0,0,RMSE,rmse,lower,0.5557362219646558,-0.5557362219646558,0.5505742164168008,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5558360123700841,-0.5558360123700841,0.5491424149361828,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p25_fr0,0,RMSE,rmse,lower,0.5555827210327928,-0.5555827210327928,0.5515477953887775,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5558022325623276,-0.5558022325623276,0.5512480144508224,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p5_fr0,0,RMSE,rmse,lower,0.5557252820018314,-0.5557252820018314,0.5515971532207388,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5558360123700841,-0.5558360123700841,0.5491424149361828,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p25_fr0,0,RMSE,rmse,lower,0.5558022325623276,-0.5558022325623276,0.5512480144508224,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5558022325623276,-0.5558022325623276,0.5512480144508224,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p5_fr0,0,RMSE,rmse,lower,0.5558360123700841,-0.5558360123700841,0.5491424149361828,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5558360123700841,-0.5558360123700841,0.5491424149361828,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p25_fr0,0,RMSE,rmse,lower,0.5560380585478252,-0.5560380585478252,0.5494440227195571,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5558022325623276,-0.5558022325623276,0.5512480144508224,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p5_fr0,0,RMSE,rmse,lower,0.5560025757843153,-0.5560025757843153,0.550425719096006,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0,False,1365 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5558360123700841,-0.5558360123700841,0.5491424149361828,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p25_fr0,0,RMSE,rmse,lower,0.5564917917545931,-0.5564917917545931,0.5481810097693468,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5558022325623276,-0.5558022325623276,0.5512480144508224,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p5_fr0,0,RMSE,rmse,lower,0.5559356776481553,-0.5559356776481553,0.55135486033622,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0,False,1365 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5558360123700841,-0.5558360123700841,0.5491424149361828,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p25_adaptert,0,RMSE,rmse,lower,0.5558750174513605,-0.5558750174513605,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p25_adapterf,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p5_adaptert,0,RMSE,rmse,lower,0.5558750174513605,-0.5558750174513605,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p5_adapterf,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p25_adaptert,0,RMSE,rmse,lower,0.5558750174513605,-0.5558750174513605,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p25_adapterf,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p5_adaptert,0,RMSE,rmse,lower,0.5558750174513605,-0.5558750174513605,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p5_adapterf,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p25_adaptert,0,RMSE,rmse,lower,0.5558750174513605,-0.5558750174513605,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p25_adapterf,0,RMSE,rmse,lower,0.5558269510040955,-0.5558269510040955,0.5527269612276788,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0,False,2730 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p5_adaptert,0,RMSE,rmse,lower,0.5558750174513605,-0.5558750174513605,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p5_adapterf,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p5_numerical_only,0,RMSE,rmse,lower,0.5558048331442785,-0.5558048331442785,0.5501619198178428,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p5_per_member,0,RMSE,rmse,lower,0.5558048331442785,-0.5558048331442785,0.5501619198178428,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p75_numerical_only,0,RMSE,rmse,lower,0.5558048331442785,-0.5558048331442785,0.5501619198178428,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p75_per_member,0,RMSE,rmse,lower,0.5558048331442785,-0.5558048331442785,0.5501619198178428,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p5_numerical_only,0,RMSE,rmse,lower,0.5558106240871751,-0.5558106240871751,0.5498568034160569,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p5_per_member,0,RMSE,rmse,lower,0.5558106240871751,-0.5558106240871751,0.5498568034160569,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p75_numerical_only,0,RMSE,rmse,lower,0.5558106240871751,-0.5558106240871751,0.5498568034160569,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p75_per_member,0,RMSE,rmse,lower,0.5558106240871751,-0.5558106240871751,0.5498568034160569,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p005_consensus_raw,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p005_softmax,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p01_consensus_raw,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p01_softmax,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p02_consensus_raw,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p02_softmax,0,RMSE,rmse,lower,0.5558749906446844,-0.5558749906446844,0.5496162918403876,mean,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p5_fr0,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p5_fr0,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p25_fr0,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p25_fr0,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p25_fr0,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p25_fr0,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p5_fr0,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p25_fr0,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p5_fr0,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p25_fr0,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p5_fr0,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0p0001_lr0p5_fr0,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n0_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r1_n1em05_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p25_fr0,0,RMSE,rmse,lower,0.5593939165294649,-0.5593939165294649,0.5608508784321002,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p25_fr0,0,RMSE,rmse,lower,0.5554104781133956,-0.5554104781133956,0.5525622419967886,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5593939165294649,-0.5593939165294649,0.5608508784321002,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5554104781133956,-0.5554104781133956,0.5525622419967886,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5596610856119687,-0.5596610856119687,0.5563749515907954,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-best-head-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p5_fr0,0,RMSE,rmse,lower,0.5596610856119687,-0.5596610856119687,0.5563749515907954,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-best-head-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5552555453020565,-0.5552555453020565,0.551575730248866,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-greedy-heads-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0_lr0p5_fr0,0,RMSE,rmse,lower,0.5552555453020565,-0.5552555453020565,0.551575730248866,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-greedy-heads-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5593939165294649,-0.5593939165294649,0.5608508784321002,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5554104781133956,-0.5554104781133956,0.5525622419967886,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p5_fr0,0,RMSE,rmse,lower,0.5583888658501774,-0.5583888658501774,0.5543243670215346,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p5_fr0,0,RMSE,rmse,lower,0.5551940861270195,-0.5551940861270195,0.5519268755870242,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5593939165294649,-0.5593939165294649,0.5608508784321002,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5554104781133956,-0.5554104781133956,0.5525622419967886,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p25_fr0,0,RMSE,rmse,lower,0.5586283212914263,-0.5586283212914263,0.5508851568810451,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p25_fr0,0,RMSE,rmse,lower,0.5550703688421238,-0.5550703688421238,0.5524078860674355,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p5_fr0,0,RMSE,rmse,lower,0.5584356978034605,-0.5584356978034605,0.5561328378795123,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p5_fr0,0,RMSE,rmse,lower,0.555272076390732,-0.555272076390732,0.5536843483543759,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p25_fr0,0,RMSE,rmse,lower,0.5603404179951016,-0.5603404179951016,0.5552489703004402,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-best-head-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p25_fr0,0,RMSE,rmse,lower,0.5554431013363782,-0.5554431013363782,0.5505825252366326,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-greedy-heads-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5596610856119687,-0.5596610856119687,0.5563749515907954,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-best-head-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n1em05_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5552555453020565,-0.5552555453020565,0.551575730248866,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5596610856119687,-0.5596610856119687,0.5563749515907954,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-best-head-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r2_n0p0001_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5552555453020565,-0.5552555453020565,0.551575730248866,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p25_fr0,0,RMSE,rmse,lower,0.5593939165294649,-0.5593939165294649,0.5608508784321002,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5593939165294649,-0.5593939165294649,0.5608508784321002,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p25_fr0,0,RMSE,rmse,lower,0.5554104781133956,-0.5554104781133956,0.5525622419967886,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5554104781133956,-0.5554104781133956,0.5525622419967886,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p5_fr0,0,RMSE,rmse,lower,0.5596610856119687,-0.5596610856119687,0.5563749515907954,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-best-head-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p5_fr0,0,RMSE,rmse,lower,0.5552555453020565,-0.5552555453020565,0.551575730248866,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-greedy-heads-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5596610856119687,-0.5596610856119687,0.5563749515907954,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-best-head-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5552555453020565,-0.5552555453020565,0.551575730248866,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-greedy-heads-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p5_fr0,0,RMSE,rmse,lower,0.5579574268466545,-0.5579574268466545,0.5527197091141781,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-best-head-evaluation/0,False,1365 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p5_fr0,0,RMSE,rmse,lower,0.5553178831836687,-0.5553178831836687,0.5514062081885815,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-greedy-heads-evaluation/0,False,1365 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5593939165294649,-0.5593939165294649,0.5608508784321002,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5554104781133956,-0.5554104781133956,0.5525622419967886,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p5_adaptert,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p5_adaptert,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p25_adaptert,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p25_adaptert,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p5_adapterf,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p5_adapterf,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p25_fr0,0,RMSE,rmse,lower,0.5601935253217213,-0.5601935253217213,0.5598375837203123,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-best-head-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p25_fr0,0,RMSE,rmse,lower,0.5553281335127394,-0.5553281335127394,0.550823697016921,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-greedy-heads-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p25_adapterf,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p00025_sf0p25_adapterf,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5596610856119687,-0.5596610856119687,0.5563749515907954,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-best-head-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n1em05_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5552555453020565,-0.5552555453020565,0.551575730248866,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p5_fr0,0,RMSE,rmse,lower,0.5572652691682626,-0.5572652691682626,0.5524207799310179,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-best-head-evaluation/0,False,1365 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p5_fr0,0,RMSE,rmse,lower,0.5553930389270275,-0.5553930389270275,0.5519483119271499,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0,False,1365 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5593939165294649,-0.5593939165294649,0.5608508784321002,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p25_fr0p5,0,RMSE,rmse,lower,0.5554104781133956,-0.5554104781133956,0.5525622419967886,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p25_adaptert,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p25_adaptert,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p5_adaptert,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p5_adaptert,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p25_adapterf,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p25_adapterf,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p5_adapterf,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p0005_sf0p5_adapterf,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p5_numerical_only,0,RMSE,rmse,lower,0.5590051856389011,-0.5590051856389011,0.5534488118080276,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p5_numerical_only,0,RMSE,rmse,lower,0.5553099940581016,-0.5553099940581016,0.5510631408064661,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p25_fr0,0,RMSE,rmse,lower,0.56004920160056,-0.56004920160056,0.5537449525211285,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-best-head-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p25_fr0,0,RMSE,rmse,lower,0.5553600907849848,-0.5553600907849848,0.5504577714884789,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5596610856119687,-0.5596610856119687,0.5563749515907954,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-best-head-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_rla_r4_n0p0001_lr0p5_fr0p5,0,RMSE,rmse,lower,0.5552555453020565,-0.5552555453020565,0.551575730248866,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0,False,3276 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p5_per_member,0,RMSE,rmse,lower,0.5590051856389011,-0.5590051856389011,0.5534488118080276,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p5_per_member,0,RMSE,rmse,lower,0.5553099940581016,-0.5553099940581016,0.5510631408064661,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p5_adaptert,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p5_adaptert,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p25_adaptert,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p25_adaptert,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p5_adapterf,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p5_adapterf,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p75_numerical_only,0,RMSE,rmse,lower,0.5590051856389011,-0.5590051856389011,0.5534488118080276,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p75_numerical_only,0,RMSE,rmse,lower,0.5553099940581016,-0.5553099940581016,0.5510631408064661,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p75_per_member,0,RMSE,rmse,lower,0.5590051856389011,-0.5590051856389011,0.5534488118080276,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p975_sf0p75_per_member,0,RMSE,rmse,lower,0.5553099940581016,-0.5553099940581016,0.5510631408064661,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p25_adapterf,0,RMSE,rmse,lower,0.5596782586864594,-0.5596782586864594,0.5565908577070521,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-best-head-evaluation/0,False,2730 +rescue_delivery_individual,delivery-eta,deliv_esam_rho0p001_sf0p25_adapterf,0,RMSE,rmse,lower,0.555452544543325,-0.555452544543325,0.5532771704027001,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-greedy-heads-evaluation/0,False,2730 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p5_numerical_only,0,RMSE,rmse,lower,0.5584120821811833,-0.5584120821811833,0.5559629363941286,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p5_numerical_only,0,RMSE,rmse,lower,0.5553168366734594,-0.5553168366734594,0.5509050649441617,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p75_numerical_only,0,RMSE,rmse,lower,0.5584120821811833,-0.5584120821811833,0.5559629363941286,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p75_numerical_only,0,RMSE,rmse,lower,0.5553168366734594,-0.5553168366734594,0.5509050649441617,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p5_per_member,0,RMSE,rmse,lower,0.5584120821811833,-0.5584120821811833,0.5559629363941286,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p5_per_member,0,RMSE,rmse,lower,0.5553168366734594,-0.5553168366734594,0.5509050649441617,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p75_per_member,0,RMSE,rmse,lower,0.5584120821811833,-0.5584120821811833,0.5559629363941286,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_mfb_k0p99_sf0p75_per_member,0,RMSE,rmse,lower,0.5553168366734594,-0.5553168366734594,0.5509050649441617,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p005_consensus_raw,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p005_consensus_raw,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p005_softmax,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p005_softmax,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p01_softmax,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p01_softmax,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p01_consensus_raw,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p01_consensus_raw,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p02_consensus_raw,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p02_consensus_raw,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-greedy-heads-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p02_softmax,0,RMSE,rmse,lower,0.5583881453279098,-0.5583881453279098,0.5557654209580059,best-head,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-best-head-evaluation/0,False,2184 +rescue_delivery_individual,delivery-eta,deliv_cf_l0p02_softmax,0,RMSE,rmse,lower,0.5553348954063525,-0.5553348954063525,0.550360417400049,greedy-heads,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-greedy-heads-evaluation/0,False,2184 +rescue_delivery_combined,delivery-eta,deliv_comb_rla_esam,0,RMSE,rmse,lower,0.5555664137600386,-0.5555664137600386,0.5512279565776417,mean,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0,False,2730 +rescue_delivery_combined,delivery-eta,deliv_comb_rla_cf,0,RMSE,rmse,lower,0.5559356776481553,-0.5559356776481553,0.55135486033622,mean,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0,False,1365 +rescue_delivery_combined,delivery-eta,deliv_comb_esam_cf,0,RMSE,rmse,lower,0.5558269510040955,-0.5558269510040955,0.5527269612276788,mean,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0,False,2730 +rescue_delivery_combined,delivery-eta,deliv_comb_rla_esam_cf,0,RMSE,rmse,lower,0.5555664137600386,-0.5555664137600386,0.5512279565776417,mean,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0,False,2730 +rescue_delivery_combined,delivery-eta,deliv_comb_esam_cf,0,RMSE,rmse,lower,0.5596782586864594,-0.5596782586864594,0.5565908577070521,best-head,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-best-head-evaluation/0,False,2730 +rescue_delivery_combined,delivery-eta,deliv_comb_esam_cf,0,RMSE,rmse,lower,0.555452544543325,-0.555452544543325,0.5532771704027001,greedy-heads,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-greedy-heads-evaluation/0,False,2730 +rescue_delivery_combined,delivery-eta,deliv_comb_rla_cf,0,RMSE,rmse,lower,0.5572652691682626,-0.5572652691682626,0.5524207799310179,best-head,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-best-head-evaluation/0,False,1365 +rescue_delivery_combined,delivery-eta,deliv_comb_rla_cf,0,RMSE,rmse,lower,0.5553930389270275,-0.5553930389270275,0.5519483119271499,greedy-heads,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-greedy-heads-evaluation/0,False,1365 +rescue_delivery_combined,delivery-eta,deliv_comb_rla_esam,0,RMSE,rmse,lower,0.5591176650506209,-0.5591176650506209,0.5583917212440949,best-head,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-best-head-evaluation/0,False,2730 +rescue_delivery_combined,delivery-eta,deliv_comb_rla_esam,0,RMSE,rmse,lower,0.5550917643214267,-0.5550917643214267,0.5526470209655567,greedy-heads,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-greedy-heads-evaluation/0,False,2730 +rescue_delivery_combined,delivery-eta,deliv_comb_rla_esam_cf,0,RMSE,rmse,lower,0.5591176650506209,-0.5591176650506209,0.5583917212440949,best-head,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-best-head-evaluation/0,False,2730 +rescue_delivery_combined,delivery-eta,deliv_comb_rla_esam_cf,0,RMSE,rmse,lower,0.5550917643214267,-0.5550917643214267,0.5526470209655567,greedy-heads,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-greedy-heads-evaluation/0,False,2730 diff --git a/tabm_integrated/paper/exp/final_integrated_summary.csv b/tabm_integrated/paper/exp/final_integrated_summary.csv index d76516a4b..9443108f7 100644 --- a/tabm_integrated/paper/exp/final_integrated_summary.csv +++ b/tabm_integrated/paper/exp/final_integrated_summary.csv @@ -1,31 +1,31 @@ -dataset,task_type,variant,source_variant,selected_config,metric,direction,metric_direction,validation_metric,test_metric,mean,std,test_mean,test_std,n_seeds,baseline_mean,baseline_std,matched_baseline_inference_mode,matched_baseline_mean,matched_baseline_std,absolute_delta,percent_delta,precision,inference_mode,selected_inference_mode,config_path,result_path,matched_baseline_result_path,status -sberbank-housing,regression,baseline_plr,baseline_plr,baseline_plr,RMSE,lower,lower,0.23552481981288087,0.23441786078206484,0.23441786078206484,0.0035482159973780638,0.23441786078206484,0.0035482159973780638,3,0.23441786078206484,0.0035482159973780638,mean,0.23441786078206484,0.0035482159973780638,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2,baseline -sberbank-housing,regression,best_rla_only,rla_rank2_noise0.001,rla_rank2_noise0.001,RMSE,lower,lower,0.23385412997377952,0.2323679570932143,0.2323679570932143,0.002285510337847783,0.2323679570932143,0.002285510337847783,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,0.0017371018906270908,0.7420180914360294,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,weak_win -sberbank-housing,regression,best_esam_only,esam_rho0.0025,esam_rho0.0025,RMSE,lower,lower,0.23381482399811443,0.23440607616465495,0.23440607616465495,0.004011724082624796,0.23440607616465495,0.004011724082624796,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,-0.00030101718081354534,-0.12858209135682216,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,loss -sberbank-housing,regression,best_mfb_only,mfb_keep0.8,mfb_keep0.8,RMSE,lower,lower,0.23321906081613947,0.23509963234240183,0.23509963234240183,0.003956610803973734,0.23509963234240183,0.003956610803973734,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,-0.00099457335856043,-0.4248406091169,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,loss -sberbank-housing,regression,best_cf_fisd_only,cf_fisd_only,cf_fisd_only,RMSE,lower,lower,0.23407921163352594,0.23309306252476805,0.23309306252476805,0.0005850816922811394,0.23309306252476805,0.0005850816922811394,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,0.00101199645907335,0.4322830371398342,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,weak_win -sberbank-housing,regression,best_combined,mfb_cf_fisd,mfb_cf_fisd,RMSE,lower,lower,0.23329192884803412,0.23497877374993395,0.23497877374993395,0.003370630932382365,0.23497877374993395,0.003370630932382365,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,-0.0008737147660925482,-0.3732148164097788,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,loss -ecom-offers,binclass,baseline_plr,baseline_plr,baseline_plr,AUROC,higher,higher,0.6214296509780997,0.5900580480375381,0.5900580480375381,0.0009703638630149601,0.5900580480375381,0.0009703638630149601,3,0.5900580480375381,0.0009703638630149601,mean,0.5900580480375381,0.0009703638630149601,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2,baseline -ecom-offers,binclass,best_rla_only,rla_rank2_noise0.0001,rla_rank2_noise0.0001,AUROC,higher,higher,0.6348279404183594,0.5984469268269463,0.5984469268269463,0.002797349140724218,0.5984469268269463,0.002797349140724218,3,0.5996938524587336,0.0035785432553676923,best-head,0.5996938524587336,0.0035785432553676923,-0.001246925631787299,-0.20792703254751188,3 seeds,best-head,best-head,exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2,loss -ecom-offers,binclass,best_esam_only,esam_rho0.001,esam_rho0.001,AUROC,higher,higher,0.6366563506545114,0.6004196058355985,0.6004196058355985,0.003493353451214683,0.6004196058355985,0.003493353451214683,3,0.6003705999706029,0.0034459818868121214,greedy-heads,0.6003705999706029,0.0034459818868121214,4.900586499556603e-05,0.008162602398912538,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2,weak_win -ecom-offers,binclass,best_mfb_only,mfb_keep0.7,mfb_keep0.7,AUROC,higher,higher,0.6539769427777475,0.6095164790194458,0.6095164790194458,0.0020894834660806928,0.6095164790194458,0.0020894834660806928,3,0.5996938524587336,0.0035785432553676923,best-head,0.5996938524587336,0.0035785432553676923,0.0098226265607122,1.637940179049629,3 seeds,best-head,best-head,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2,clear_win -ecom-offers,binclass,best_cf_fisd_only,cf_fisd_lambda0.05,cf_fisd_lambda0.05,AUROC,higher,higher,0.636440295793426,0.6002048966489212,0.6002048966489212,0.0034582462660955543,0.6002048966489212,0.0034582462660955543,3,0.6003705999706029,0.0034459818868121214,greedy-heads,0.6003705999706029,0.0034459818868121214,-0.00016570332168175383,-0.02760017257505072,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2,loss -ecom-offers,binclass,best_combined,mfb_cf_fisd,mfb_cf_fisd,AUROC,higher,higher,0.6485201184226032,0.6052109584242933,0.6052109584242933,0.003222479623952224,0.6052109584242933,0.003222479623952224,3,0.6003705999706029,0.0034459818868121214,greedy-heads,0.6003705999706029,0.0034459818868121214,0.004840358453690352,0.8062284285618516,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml,exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2,clear_win -homesite-insurance,binclass,baseline_plr,baseline_plr,baseline_plr,AUROC,higher,higher,0.9584198249724393,0.9624941853379376,0.9624941853379376,0.0001780794332642721,0.9624941853379376,0.0001780794332642721,3,0.9624941853379376,0.0001780794332642721,mean,0.9624941853379376,0.0001780794332642721,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2,baseline -homesite-insurance,binclass,best_rla_only,rla_rank4_noise1e-05,rla_rank4_noise1e-05,AUROC,higher,higher,0.9589784960899528,0.9625640466562138,0.9625640466562138,0.0006085287596608143,0.9625640466562138,0.0006085287596608143,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,-0.00011150507503732143,-0.011582830252289419,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,loss -homesite-insurance,binclass,best_esam_only,esam_only,esam_only,AUROC,higher,higher,0.959080906574986,0.9627318700697393,0.9627318700697393,0.0004492301463849077,0.9627318700697393,0.0004492301463849077,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,5.6318338488181574e-05,0.005850188922623008,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,weak_win -homesite-insurance,binclass,best_mfb_only,mfb_keep0.95,mfb_keep0.95,AUROC,higher,higher,0.959182264937648,0.962382329909275,0.962382329909275,0.0004577202193422504,0.962382329909275,0.0004577202193422504,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,-0.00029322182197610847,-0.030459049411692845,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,loss -homesite-insurance,binclass,best_cf_fisd_only,cf_fisd_only,cf_fisd_only,AUROC,higher,higher,0.959527988245724,0.9629076428862089,0.9629076428862089,0.00047402677133884024,0.9629076428862089,0.00047402677133884024,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,0.00023209115495781418,0.024108969479948606,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,weak_win -homesite-insurance,binclass,best_combined,mfb_cf_fisd,mfb_cf_fisd,AUROC,higher,higher,0.95963296602657,0.9628490210733115,0.9628490210733115,0.0004379684704485265,0.9628490210733115,0.0004379684704485265,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,0.0001734693420604172,0.018019502183103577,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,weak_win -cooking-time,regression,baseline_plr,baseline_plr,baseline_plr,RMSE,lower,lower,0.46270875116661525,0.4803806470025843,0.4803806470025843,0.00012338540954176603,0.4803806470025843,0.00012338540954176603,3,0.4803806470025843,0.00012338540954176603,mean,0.4803806470025843,0.00012338540954176603,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml,exp/integrated/final/cooking-time/baseline_plr-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-evaluation/2,baseline -cooking-time,regression,best_rla_only,rla_rank4_noise0.001,rla_rank4_noise0.001,RMSE,lower,lower,0.46244170136784024,0.48014625604758127,0.48014625604758127,0.00016088993704992786,0.48014625604758127,0.00016088993704992786,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,-2.08738622038962e-05,-0.004347585647083486,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,loss -cooking-time,regression,best_esam_only,esam_rho0.0025,esam_rho0.0025,RMSE,lower,lower,0.46247755253655587,0.4800648691653293,0.4800648691653293,8.021206715574865e-05,0.4800648691653293,8.021206715574865e-05,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,6.0513020048058586e-05,0.012603586957353233,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,weak_win -cooking-time,regression,best_mfb_only,mfb_keep0.8,mfb_keep0.8,RMSE,lower,lower,0.4625031808631155,0.4799728414668436,0.4799728414668436,0.00019455249694504936,0.4799728414668436,0.00019455249694504936,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,0.00015254071853376683,0.03177101736205868,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,weak_win -cooking-time,regression,best_cf_fisd_only,cf_fisd_lambda0.05,cf_fisd_lambda0.05,RMSE,lower,lower,0.46295431080753496,0.48033743363756964,0.48033743363756964,4.649528370692734e-05,0.48033743363756964,4.649528370692734e-05,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,-0.0002120514521922745,-0.04416584918445345,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,loss -cooking-time,regression,best_combined,rla_esam,rla_esam,RMSE,lower,lower,0.46244255150399494,0.48002605538016,0.48002605538016,9.65539747406379e-05,0.48002605538016,9.65539747406379e-05,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,9.932680521734794e-05,0.020687680531540333,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_combined-evaluation/0.toml;exp/integrated/final/cooking-time/best_combined-evaluation/1.toml;exp/integrated/final/cooking-time/best_combined-evaluation/2.toml,exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,weak_win -delivery-eta,regression,baseline_plr,baseline_plr,baseline_plr,RMSE,lower,lower,0.5559497041253089,0.5502258480608934,0.5502258480608934,0.0005822921162605647,0.5502258480608934,0.0005822921162605647,3,0.5502258480608934,0.0005822921162605647,mean,0.5502258480608934,0.0005822921162605647,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2,baseline -delivery-eta,regression,best_rla_only,rla_rank2_noise0.001,rla_rank2_noise0.001,RMSE,lower,lower,0.5556523766610589,0.5514065614560619,0.5514065614560619,0.001097158835638919,0.5514065614560619,0.001097158835638919,3,0.5511744515458483,0.0007068650190236373,greedy-heads,0.5511744515458483,0.0007068650190236373,-0.00023210991021360705,-0.04211187756664361,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2,loss -delivery-eta,regression,best_esam_only,esam_only,esam_only,RMSE,lower,lower,0.5554227463560124,0.5511479242060323,0.5511479242060323,0.0005047191238214393,0.5511479242060323,0.0005047191238214393,3,0.5511744515458483,0.0007068650190236373,greedy-heads,0.5511744515458483,0.0007068650190236373,2.652733981600708e-05,0.004812875441088992,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2,weak_win -delivery-eta,regression,best_mfb_only,mfb_keep0.7,mfb_keep0.7,RMSE,lower,lower,0.5550787103394562,0.551366165184366,0.551366165184366,0.0017138685603590752,0.551366165184366,0.0017138685603590752,3,0.5511744515458483,0.0007068650190236373,greedy-heads,0.5511744515458483,0.0007068650190236373,-0.00019171363851766543,-0.03478275126504446,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2,loss -delivery-eta,regression,best_cf_fisd_only,cf_fisd_lambda0.05,cf_fisd_lambda0.05,RMSE,lower,lower,0.5558348472452875,0.5524340881807381,0.5524340881807381,0.0013911659814170834,0.5524340881807381,0.0013911659814170834,3,0.5511744515458483,0.0007068650190236373,greedy-heads,0.5511744515458483,0.0007068650190236373,-0.0012596366348898158,-0.22853683282252707,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2,loss -delivery-eta,regression,best_combined,rla_esam,rla_esam,RMSE,lower,lower,0.5558806192547581,0.5523362750771902,0.5523362750771902,0.00019897426137676394,0.5523362750771902,0.00019897426137676394,3,0.5511744515458483,0.0007068650190236373,greedy-heads,0.5511744515458483,0.0007068650190236373,-0.0011618235313418657,-0.21079052704336418,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml,exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2,loss +dataset,task_type,variant,source_variant,selected_config,metric,direction,metric_direction,validation_metric,matched_validation_baseline,matched_validation_delta,matched_validation_percent_delta,test_metric,mean,std,test_mean,test_std,n_seeds,baseline_mean,baseline_std,matched_baseline_inference_mode,matched_baseline_mean,matched_baseline_std,absolute_delta,percent_delta,precision,inference_mode,selected_inference_mode,config_path,result_path,matched_baseline_result_path,status +sberbank-housing,regression,baseline_plr,baseline_plr,baseline_plr,RMSE,lower,lower,0.23552481981288087,0.23552481981288087,0.0,0.0,0.23441786078206484,0.23441786078206484,0.0035482159973780638,0.23441786078206484,0.0035482159973780638,3,0.23441786078206484,0.0035482159973780638,mean,0.23441786078206484,0.0035482159973780638,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2,baseline +sberbank-housing,regression,best_rla_only,rla_rank2_noise0.001,rla_rank2_noise0.001,RMSE,lower,lower,0.23385412997377952,0.23375674885076472,-9.738112301480029e-05,-0.041659170694990484,0.2323679570932143,0.2323679570932143,0.002285510337847783,0.2323679570932143,0.002285510337847783,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,0.0017371018906270908,0.7420180914360294,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_rla_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_rla_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_rla_only-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,weak_win +sberbank-housing,regression,best_esam_only,esam_rho0.0025,esam_rho0.0025,RMSE,lower,lower,0.23381482399811443,0.23375674885076472,-5.8075147349706535e-05,-0.02484426551756285,0.23440607616465495,0.23440607616465495,0.004011724082624796,0.23440607616465495,0.004011724082624796,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,-0.00030101718081354534,-0.12858209135682216,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,loss +sberbank-housing,regression,best_mfb_only,mfb_keep0.8,mfb_keep0.8,RMSE,lower,lower,0.23321906081613947,0.23375674885076472,0.0005376880346252555,0.23002032551732954,0.23509963234240183,0.23509963234240183,0.003956610803973734,0.23509963234240183,0.003956610803973734,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,-0.00099457335856043,-0.4248406091169,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,loss +sberbank-housing,regression,best_cf_fisd_only,cf_fisd_only,cf_fisd_only,RMSE,lower,lower,0.23407921163352594,0.23375674885076472,-0.00032246278276121254,-0.137948009777925,0.23309306252476805,0.23309306252476805,0.0005850816922811394,0.23309306252476805,0.0005850816922811394,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,0.00101199645907335,0.4322830371398342,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,weak_win +sberbank-housing,regression,best_combined,mfb_cf_fisd,mfb_cf_fisd,RMSE,lower,lower,0.23329192884803412,0.23375674885076472,0.0004648200027305982,0.1988477359545025,0.23497877374993395,0.23497877374993395,0.003370630932382365,0.23497877374993395,0.003370630932382365,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,-0.0008737147660925482,-0.3732148164097788,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,loss +ecom-offers,binclass,baseline_plr,baseline_plr,baseline_plr,AUROC,higher,higher,0.6214296509780997,0.6214296509780997,0.0,0.0,0.5900580480375381,0.5900580480375381,0.0009703638630149601,0.5900580480375381,0.0009703638630149601,3,0.5900580480375381,0.0009703638630149601,mean,0.5900580480375381,0.0009703638630149601,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2,baseline +ecom-offers,binclass,best_rla_only,rla_rank2_noise0.0001,rla_rank2_noise0.0001,AUROC,higher,higher,0.6348279404183594,0.635345130387201,-0.000517189968841536,-0.08140299564841912,0.5984469268269463,0.5984469268269463,0.002797349140724218,0.5984469268269463,0.002797349140724218,3,0.5996938524587336,0.0035785432553676923,best-head,0.5996938524587336,0.0035785432553676923,-0.001246925631787299,-0.20792703254751188,3 seeds,best-head,best-head,exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2,loss +ecom-offers,binclass,best_esam_only,esam_rho0.001,esam_rho0.001,AUROC,higher,higher,0.6366563506545114,0.6365904008411623,6.594981334917538e-05,0.0103598504253335,0.6004196058355985,0.6004196058355985,0.003493353451214683,0.6004196058355985,0.003493353451214683,3,0.6003705999706029,0.0034459818868121214,greedy-heads,0.6003705999706029,0.0034459818868121214,4.900586499556603e-05,0.008162602398912538,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2,weak_win +ecom-offers,binclass,best_mfb_only,mfb_keep0.7,mfb_keep0.7,AUROC,higher,higher,0.6539769427777475,0.635345130387201,0.018631812390546565,2.9325498063063304,0.6095164790194458,0.6095164790194458,0.0020894834660806928,0.6095164790194458,0.0020894834660806928,3,0.5996938524587336,0.0035785432553676923,best-head,0.5996938524587336,0.0035785432553676923,0.0098226265607122,1.637940179049629,3 seeds,best-head,best-head,exp/integrated/final/ecom-offers/best_mfb_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_mfb_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_mfb_only-best-head-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2,clear_win +ecom-offers,binclass,best_cf_fisd_only,cf_fisd_lambda0.05,cf_fisd_lambda0.05,AUROC,higher,higher,0.636440295793426,0.6365904008411623,-0.00015010504773627087,-0.02357953364328597,0.6002048966489212,0.6002048966489212,0.0034582462660955543,0.6002048966489212,0.0034582462660955543,3,0.6003705999706029,0.0034459818868121214,greedy-heads,0.6003705999706029,0.0034459818868121214,-0.00016570332168175383,-0.02760017257505072,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2,loss +ecom-offers,binclass,best_combined,mfb_cf_fisd,mfb_cf_fisd,AUROC,higher,higher,0.6485201184226032,0.6365904008411623,0.011929717581440902,1.8740021159096183,0.6052109584242933,0.6052109584242933,0.003222479623952224,0.6052109584242933,0.003222479623952224,3,0.6003705999706029,0.0034459818868121214,greedy-heads,0.6003705999706029,0.0034459818868121214,0.004840358453690352,0.8062284285618516,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/ecom-offers/best_combined-evaluation/0.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/1.toml;exp/integrated/final/ecom-offers/best_combined-evaluation/2.toml,exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_combined-greedy-heads-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2,clear_win +homesite-insurance,binclass,baseline_plr,baseline_plr,baseline_plr,AUROC,higher,higher,0.9584198249724393,0.9584198249724393,0.0,0.0,0.9624941853379376,0.9624941853379376,0.0001780794332642721,0.9624941853379376,0.0001780794332642721,3,0.9624941853379376,0.0001780794332642721,mean,0.9624941853379376,0.0001780794332642721,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1.toml;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2.toml,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-evaluation/2,baseline +homesite-insurance,binclass,best_rla_only,rla_rank4_noise1e-05,rla_rank4_noise1e-05,AUROC,higher,higher,0.9589784960899528,0.9591396202785444,-0.00016112418859159927,-0.01679882523722741,0.9625640466562138,0.9625640466562138,0.0006085287596608143,0.9625640466562138,0.0006085287596608143,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,-0.00011150507503732143,-0.011582830252289419,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_rla_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_rla_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_rla_only-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,loss +homesite-insurance,binclass,best_esam_only,esam_only,esam_only,AUROC,higher,higher,0.959080906574986,0.9591396202785444,-5.871370355836625e-05,-0.006121497049753315,0.9627318700697393,0.9627318700697393,0.0004492301463849077,0.9627318700697393,0.0004492301463849077,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,5.6318338488181574e-05,0.005850188922623008,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_esam_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_esam_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,weak_win +homesite-insurance,binclass,best_mfb_only,mfb_keep0.95,mfb_keep0.95,AUROC,higher,higher,0.959182264937648,0.9591396202785444,4.2644659103641125e-05,0.004446136746103415,0.962382329909275,0.962382329909275,0.0004577202193422504,0.962382329909275,0.0004577202193422504,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,-0.00029322182197610847,-0.030459049411692845,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_mfb_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_mfb_only-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,loss +homesite-insurance,binclass,best_cf_fisd_only,cf_fisd_only,cf_fisd_only,AUROC,higher,higher,0.959527988245724,0.9591396202785444,0.00038836796717967026,0.04049128604101289,0.9629076428862089,0.9629076428862089,0.00047402677133884024,0.9629076428862089,0.00047402677133884024,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,0.00023209115495781418,0.024108969479948606,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,weak_win +homesite-insurance,binclass,best_combined,mfb_cf_fisd,mfb_cf_fisd,AUROC,higher,higher,0.95963296602657,0.9591396202785444,0.0004933457480256198,0.05143628076612526,0.9628490210733115,0.9628490210733115,0.0004379684704485265,0.9628490210733115,0.0004379684704485265,3,0.9626755517312511,0.00027672454564782147,greedy-heads,0.9626755517312511,0.00027672454564782147,0.0001734693420604172,0.018019502183103577,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/homesite-insurance/best_combined-evaluation/0.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/1.toml;exp/integrated/final/homesite-insurance/best_combined-evaluation/2.toml,exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/best_combined-greedy-heads-evaluation/2,exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/homesite-insurance/baseline_plr-greedy-heads-evaluation/2,weak_win +cooking-time,regression,baseline_plr,baseline_plr,baseline_plr,RMSE,lower,lower,0.46270875116661525,0.46270875116661525,0.0,0.0,0.4803806470025843,0.4803806470025843,0.00012338540954176603,0.4803806470025843,0.00012338540954176603,3,0.4803806470025843,0.00012338540954176603,mean,0.4803806470025843,0.00012338540954176603,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/cooking-time/baseline_plr-evaluation/0.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/1.toml;exp/integrated/final/cooking-time/baseline_plr-evaluation/2.toml,exp/integrated/final/cooking-time/baseline_plr-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-evaluation/2,baseline +cooking-time,regression,best_rla_only,rla_rank4_noise0.001,rla_rank4_noise0.001,RMSE,lower,lower,0.46244170136784024,0.4624700316244145,2.8330256574282675e-05,0.006125857814996866,0.48014625604758127,0.48014625604758127,0.00016088993704992786,0.48014625604758127,0.00016088993704992786,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,-2.08738622038962e-05,-0.004347585647083486,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_rla_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_rla_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_rla_only-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,loss +cooking-time,regression,best_esam_only,esam_rho0.0025,esam_rho0.0025,RMSE,lower,lower,0.46247755253655587,0.4624700316244145,-7.520912141345537e-06,-0.0016262485408899945,0.4800648691653293,0.4800648691653293,8.021206715574865e-05,0.4800648691653293,8.021206715574865e-05,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,6.0513020048058586e-05,0.012603586957353233,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_esam_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_esam_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,weak_win +cooking-time,regression,best_mfb_only,mfb_keep0.8,mfb_keep0.8,RMSE,lower,lower,0.4625031808631155,0.4624700316244145,-3.3149238700957984e-05,-0.007167867415002461,0.4799728414668436,0.4799728414668436,0.00019455249694504936,0.4799728414668436,0.00019455249694504936,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,0.00015254071853376683,0.03177101736205868,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_mfb_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_mfb_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_mfb_only-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,weak_win +cooking-time,regression,best_cf_fisd_only,cf_fisd_lambda0.05,cf_fisd_lambda0.05,RMSE,lower,lower,0.46295431080753496,0.4624700316244145,-0.0004842791831204396,-0.10471579778249003,0.48033743363756964,0.48033743363756964,4.649528370692734e-05,0.48033743363756964,4.649528370692734e-05,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,-0.0002120514521922745,-0.04416584918445345,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/cooking-time/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,loss +cooking-time,regression,best_combined,rla_esam,rla_esam,RMSE,lower,lower,0.46244255150399494,0.4624700316244145,2.748012041958736e-05,0.005942032681137007,0.48002605538016,0.48002605538016,9.65539747406379e-05,0.48002605538016,9.65539747406379e-05,3,0.48012538218537737,0.0002586745994143849,greedy-heads,0.48012538218537737,0.0002586745994143849,9.932680521734794e-05,0.020687680531540333,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/cooking-time/best_combined-evaluation/0.toml;exp/integrated/final/cooking-time/best_combined-evaluation/1.toml;exp/integrated/final/cooking-time/best_combined-evaluation/2.toml,exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/best_combined-greedy-heads-evaluation/2,exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/cooking-time/baseline_plr-greedy-heads-evaluation/2,weak_win +delivery-eta,regression,baseline_plr,baseline_plr,baseline_plr,RMSE,lower,lower,0.5559497041253089,0.5559497041253089,0.0,0.0,0.5502258480608934,0.5502258480608934,0.0005822921162605647,0.5502258480608934,0.0005822921162605647,3,0.5502258480608934,0.0005822921162605647,mean,0.5502258480608934,0.0005822921162605647,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1.toml;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2.toml,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2,baseline +delivery-eta,regression,best_rla_only,deliv_rla_r4_n0p0001_lr0p5_fr0,deliv_rla_r4_n0p0001_lr0p5_fr0,RMSE,lower,lower,0.557867731128666,0.5579635797458202,9.584861715428161e-05,0.017178292747699653,0.5543336992032142,0.5543336992032142,0.002661909801131988,0.5543336992032142,0.002661909801131988,3,0.5566895883615489,0.000859735360596188,best-head,0.5566895883615489,0.000859735360596188,0.002355889158334734,0.42319619543605924,3 seeds,best-head,best-head,exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0;exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1;exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/2,clear_win +delivery-eta,regression,best_esam_only,deliv_esam_rho0p001_sf0p25_adapterf,deliv_esam_rho0p001_sf0p25_adapterf,RMSE,lower,lower,0.5559336909117792,0.5559497041253089,1.601321352961982e-05,0.002880334931522961,0.5512627378566571,0.5512627378566571,0.0012916491703925872,0.5512627378566571,0.0012916491703925872,3,0.5502258480608934,0.0005822921162605647,mean,0.5502258480608934,0.0005822921162605647,-0.0010368897957637735,-0.18844803446039146,3 seeds,mean,mean,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_esam_only-evaluation/0;exp/integrated/final/delivery-eta/best_esam_only-evaluation/1;exp/integrated/final/delivery-eta/best_esam_only-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2,loss +delivery-eta,regression,best_mfb_only,deliv_mfb_k0p975_sf0p5_numerical_only,deliv_mfb_k0p975_sf0p5_numerical_only,RMSE,lower,lower,0.5560023662408545,0.5559497041253089,-5.2662115545665245e-05,-0.009472460396128821,0.5501995737833563,0.5501995737833563,0.0001818929992957081,0.5501995737833563,0.0001818929992957081,3,0.5502258480608934,0.0005822921162605647,mean,0.5502258480608934,0.0005822921162605647,2.6274277537097213e-05,0.004775180524450652,3 seeds,mean,mean,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1;exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-evaluation/2,weak_win +delivery-eta,regression,best_cf_fisd_only,deliv_cf_l0p005_consensus_raw,deliv_cf_l0p005_consensus_raw,RMSE,lower,lower,0.5553874207148172,0.5553874207148172,0.0,0.0,0.5511744515458483,0.5511744515458483,0.0007068650190236373,0.5511744515458483,0.0007068650190236373,3,0.5511744515458483,0.0007068650190236373,greedy-heads,0.5511744515458483,0.0007068650190236373,0.0,0.0,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-greedy-heads-evaluation/2,tie +delivery-eta,regression,best_combined,deliv_comb_rla_cf,deliv_comb_rla_cf,RMSE,lower,lower,0.557867731128666,0.5579635797458202,9.584861715428161e-05,0.017178292747699653,0.5543336992032142,0.5543336992032142,0.002661909801131988,0.5543336992032142,0.002661909801131988,3,0.5566895883615489,0.000859735360596188,best-head,0.5566895883615489,0.000859735360596188,0.002355889158334734,0.42319619543605924,3 seeds,best-head,best-head,exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml;exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml,exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0;exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1;exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2,exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/0;exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/1;exp/integrated/final/delivery-eta/baseline_plr-best-head-evaluation/2,clear_win diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/0/report.json index 0927cc2e8..dd789361b 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/0/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/0/report.json @@ -38,7 +38,7 @@ "n_bins": 81 }, "cf_fisd": { - "lambda": 0.05, + "lambda": 0.005, "variant": "raw", "dataset_name": "delivery-eta", "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", @@ -46,7 +46,114 @@ "xgb", "lgbm", "cat" - ] + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } } }, "n_parameters": 6761248, @@ -59,7 +166,7 @@ } }, "cf_fisd": { - "lambda": 0.05, + "lambda": 0.005, "variant": "raw", "teacher_names": [ "xgb", @@ -78,9 +185,41 @@ 7, 8, 9, - 10 + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 ], "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, 11, 12, 13, @@ -91,9 +230,41 @@ 18, 19, 20, - 21 + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 ], "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, 22, 23, 24, @@ -329,7 +500,9 @@ 2 ], "n_features": 220, - "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" }, "epoch_size": 273, "ncl": { @@ -404,46 +577,46 @@ "best_step": 2184, "metrics": { "train": { - "rmse": 0.5140202432442316, - "mae": 0.40386128425598145, - "r2": 0.5047702789306641, - "score": -0.5140202432442316 + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 }, "val": { - "rmse": 0.557661572508718, - "mae": 0.44313740730285645, - "r2": 0.4135657548904419, - "score": -0.557661572508718 + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 }, "test": { - "rmse": 0.553144673358601, - "mae": 0.44622060656547546, - "r2": 0.42985349893569946, - "score": -0.553144673358601 + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 } }, - "time": "0:08:00.261340", + "time": "0:07:43.023818", "diversity": { "train": { - "mean_centered_corr": -0.01827983371913433, + "mean_centered_corr": -0.020328260958194733, "mean_pairwise_disagreement": NaN, - "member_std": 0.0643538236618042 + "member_std": 0.06901570409536362 }, "val": { - "mean_centered_corr": -0.018152762204408646, + "mean_centered_corr": -0.020442906767129898, "mean_pairwise_disagreement": NaN, - "member_std": 0.06669643521308899 + "member_std": 0.07035460323095322 }, "test": { - "mean_centered_corr": -0.018407996743917465, + "mean_centered_corr": -0.020237648859620094, "mean_pairwise_disagreement": NaN, - "member_std": 0.06605677306652069 + "member_std": 0.06958991289138794 } }, "chunk_size": null, "eval_batch_size": 16384, "heads": [ - 30 + 13 ], - "head_selection_time": "0:00:00.023441" + "head_selection_time": "0:00:00.023628" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/1/report.json index 4a2e2847e..5dd192787 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/1/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/1/report.json @@ -38,7 +38,7 @@ "n_bins": 81 }, "cf_fisd": { - "lambda": 0.05, + "lambda": 0.005, "variant": "raw", "dataset_name": "delivery-eta", "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", @@ -46,7 +46,114 @@ "xgb", "lgbm", "cat" - ] + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } } }, "n_parameters": 6761248, @@ -59,7 +166,7 @@ } }, "cf_fisd": { - "lambda": 0.05, + "lambda": 0.005, "variant": "raw", "teacher_names": [ "xgb", @@ -78,9 +185,41 @@ 7, 8, 9, - 10 + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 ], "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, 11, 12, 13, @@ -91,9 +230,41 @@ 18, 19, 20, - 21 + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 ], "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, 22, 23, 24, @@ -329,7 +500,9 @@ 2 ], "n_features": 220, - "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" }, "epoch_size": 273, "ncl": { @@ -401,49 +574,49 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 1365, + "best_step": 1638, "metrics": { "train": { - "rmse": 0.5190214940362947, - "mae": 0.4106391668319702, - "r2": 0.49508655071258545, - "score": -0.5190214940362947 + "rmse": 0.5157048423679549, + "mae": 0.408806711435318, + "r2": 0.5015189051628113, + "score": -0.5157048423679549 }, "val": { - "rmse": 0.5570080264093609, - "mae": 0.441851943731308, - "r2": 0.4149394631385803, - "score": -0.5570080264093609 + "rmse": 0.5578575885405295, + "mae": 0.44588109850883484, + "r2": 0.41315340995788574, + "score": -0.5578575885405295 }, "test": { - "rmse": 0.5540194450901057, - "mae": 0.44831377267837524, - "r2": 0.4280487298965454, - "score": -0.5540194450901057 + "rmse": 0.5568376961917461, + "mae": 0.4522704780101776, + "r2": 0.4222150444984436, + "score": -0.5568376961917461 } }, - "time": "0:07:03.676796", + "time": "0:07:06.135890", "diversity": { "train": { - "mean_centered_corr": -0.0186633188277483, + "mean_centered_corr": -0.02029341645538807, "mean_pairwise_disagreement": NaN, - "member_std": 0.056279826909303665 + "member_std": 0.06168676167726517 }, "val": { - "mean_centered_corr": -0.018989257514476776, + "mean_centered_corr": -0.020232515409588814, "mean_pairwise_disagreement": NaN, - "member_std": 0.05482005700469017 + "member_std": 0.0631292313337326 }, "test": { - "mean_centered_corr": -0.019170163199305534, + "mean_centered_corr": -0.020325176417827606, "mean_pairwise_disagreement": NaN, - "member_std": 0.05448836833238602 + "member_std": 0.061680618673563004 } }, "chunk_size": null, "eval_batch_size": 16384, "heads": [ - 30 + 22 ], - "head_selection_time": "0:00:00.023472" + "head_selection_time": "0:00:00.023866" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/2/report.json index 98f0c74f5..ac1798ed1 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/2/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-best-head-evaluation/2/report.json @@ -38,7 +38,7 @@ "n_bins": 81 }, "cf_fisd": { - "lambda": 0.05, + "lambda": 0.005, "variant": "raw", "dataset_name": "delivery-eta", "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", @@ -46,7 +46,114 @@ "xgb", "lgbm", "cat" - ] + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } } }, "n_parameters": 6761248, @@ -59,7 +166,7 @@ } }, "cf_fisd": { - "lambda": 0.05, + "lambda": 0.005, "variant": "raw", "teacher_names": [ "xgb", @@ -78,9 +185,41 @@ 7, 8, 9, - 10 + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 ], "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, 11, 12, 13, @@ -91,9 +230,41 @@ 18, 19, 20, - 21 + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 ], "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, 22, 23, 24, @@ -329,7 +500,9 @@ 2 ], "n_features": 220, - "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" }, "epoch_size": 273, "ncl": { @@ -401,49 +574,49 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 1365, + "best_step": 1638, "metrics": { "train": { - "rmse": 0.5183413941542616, - "mae": 0.4073716104030609, - "r2": 0.4964088797569275, - "score": -0.5183413941542616 + "rmse": 0.5148308736438392, + "mae": 0.40706372261047363, + "r2": 0.5032070279121399, + "score": -0.5148308736438392 }, "val": { - "rmse": 0.5578165315968546, - "mae": 0.4420972168445587, - "r2": 0.4132397770881653, - "score": -0.5578165315968546 + "rmse": 0.5576450053690213, + "mae": 0.4445996880531311, + "r2": 0.4136006236076355, + "score": -0.5576450053690213 }, "test": { - "rmse": 0.5527351028922852, - "mae": 0.44726672768592834, - "r2": 0.430697500705719, - "score": -0.5527351028922852 + "rmse": 0.5574656479348947, + "mae": 0.45258787274360657, + "r2": 0.42091113328933716, + "score": -0.5574656479348947 } }, - "time": "0:07:03.426340", + "time": "0:07:04.841597", "diversity": { "train": { - "mean_centered_corr": -0.018950974568724632, + "mean_centered_corr": -0.0205259807407856, "mean_pairwise_disagreement": NaN, - "member_std": 0.060403864830732346 + "member_std": 0.06119334325194359 }, "val": { - "mean_centered_corr": -0.019536394625902176, + "mean_centered_corr": -0.020698808133602142, "mean_pairwise_disagreement": NaN, - "member_std": 0.06044872850179672 + "member_std": 0.06522355228662491 }, "test": { - "mean_centered_corr": -0.01978791505098343, + "mean_centered_corr": -0.020667163655161858, "mean_pairwise_disagreement": NaN, - "member_std": 0.06026185676455498 + "member_std": 0.06273415684700012 } }, "chunk_size": null, "eval_batch_size": 16384, "heads": [ - 24 + 8 ], - "head_selection_time": "0:00:00.023048" + "head_selection_time": "0:00:00.023514" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml index f7bf33592..e7d7fb4f3 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml @@ -33,7 +33,7 @@ d_embedding = 32 n_bins = 81 [cf_fisd] -lambda = 0.05 +lambda = 0.005 variant = "raw" dataset_name = "delivery-eta" teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" @@ -42,3 +42,110 @@ teacher_names = [ "lgbm", "cat", ] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0/report.json index 8dc921357..c88d18129 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0/report.json @@ -38,7 +38,7 @@ "n_bins": 81 }, "cf_fisd": { - "lambda": 0.05, + "lambda": 0.005, "variant": "raw", "dataset_name": "delivery-eta", "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", @@ -46,7 +46,114 @@ "xgb", "lgbm", "cat" - ] + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } } }, "n_parameters": 6761248, @@ -59,7 +166,7 @@ } }, "cf_fisd": { - "lambda": 0.05, + "lambda": 0.005, "variant": "raw", "teacher_names": [ "xgb", @@ -78,9 +185,41 @@ 7, 8, 9, - 10 + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 ], "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, 11, 12, 13, @@ -91,9 +230,41 @@ 18, 19, 20, - 21 + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 ], "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, 22, 23, 24, @@ -329,7 +500,9 @@ 2 ], "n_features": 220, - "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" }, "epoch_size": 273, "ncl": { @@ -404,40 +577,40 @@ "best_step": 2184, "metrics": { "train": { - "rmse": 0.5080552327962835, - "mae": 0.3986222445964813, - "r2": 0.5161974430084229, - "score": -0.5080552327962835 + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 }, "val": { - "rmse": 0.5562445854609275, - "mae": 0.4416752755641937, - "r2": 0.4165421724319458, - "score": -0.5562445854609275 + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 }, "test": { - "rmse": 0.5504166498793033, - "mae": 0.44359084963798523, - "r2": 0.4354633688926697, - "score": -0.5504166498793033 + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 } }, - "time": "0:08:00.261340", + "time": "0:07:43.023818", "diversity": { "train": { - "mean_centered_corr": -0.01827983371913433, + "mean_centered_corr": -0.020328260958194733, "mean_pairwise_disagreement": NaN, - "member_std": 0.0643538236618042 + "member_std": 0.06901570409536362 }, "val": { - "mean_centered_corr": -0.018152762204408646, + "mean_centered_corr": -0.020442906767129898, "mean_pairwise_disagreement": NaN, - "member_std": 0.06669643521308899 + "member_std": 0.07035460323095322 }, "test": { - "mean_centered_corr": -0.018407996743917465, + "mean_centered_corr": -0.020237648859620094, "mean_pairwise_disagreement": NaN, - "member_std": 0.06605677306652069 + "member_std": 0.06958991289138794 } }, "chunk_size": null, diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml index 18508bd65..59de1c3ce 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml @@ -33,7 +33,7 @@ d_embedding = 32 n_bins = 81 [cf_fisd] -lambda = 0.05 +lambda = 0.005 variant = "raw" dataset_name = "delivery-eta" teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" @@ -42,3 +42,110 @@ teacher_names = [ "lgbm", "cat", ] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1/report.json index 3e98b4d75..c6460e960 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1/report.json @@ -38,7 +38,7 @@ "n_bins": 81 }, "cf_fisd": { - "lambda": 0.05, + "lambda": 0.005, "variant": "raw", "dataset_name": "delivery-eta", "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", @@ -46,7 +46,114 @@ "xgb", "lgbm", "cat" - ] + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } } }, "n_parameters": 6761248, @@ -59,7 +166,7 @@ } }, "cf_fisd": { - "lambda": 0.05, + "lambda": 0.005, "variant": "raw", "teacher_names": [ "xgb", @@ -78,9 +185,41 @@ 7, 8, 9, - 10 + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 ], "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, 11, 12, 13, @@ -91,9 +230,41 @@ 18, 19, 20, - 21 + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 ], "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, 22, 23, 24, @@ -329,7 +500,9 @@ 2 ], "n_features": 220, - "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" }, "epoch_size": 273, "ncl": { @@ -401,43 +574,43 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 1365, + "best_step": 1638, "metrics": { "train": { - "rmse": 0.5155687012426072, - "mae": 0.40611687302589417, - "r2": 0.5017820596694946, - "score": -0.5155687012426072 + "rmse": 0.5105330146478386, + "mae": 0.40035760402679443, + "r2": 0.5114669799804688, + "score": -0.5105330146478386 }, "val": { - "rmse": 0.5564364677950075, - "mae": 0.44130873680114746, - "r2": 0.41613954305648804, - "score": -0.5564364677950075 + "rmse": 0.5562247345576334, + "mae": 0.439607709646225, + "r2": 0.416583776473999, + "score": -0.5562247345576334 }, "test": { - "rmse": 0.550973168882935, - "mae": 0.4452160894870758, - "r2": 0.43432122468948364, - "score": -0.550973168882935 + "rmse": 0.550284872190118, + "mae": 0.44156232476234436, + "r2": 0.4357336759567261, + "score": -0.550284872190118 } }, - "time": "0:07:03.676796", + "time": "0:07:06.135890", "diversity": { "train": { - "mean_centered_corr": -0.0186633188277483, + "mean_centered_corr": -0.02029341645538807, "mean_pairwise_disagreement": NaN, - "member_std": 0.056279826909303665 + "member_std": 0.06168676167726517 }, "val": { - "mean_centered_corr": -0.018989257514476776, + "mean_centered_corr": -0.020232515409588814, "mean_pairwise_disagreement": NaN, - "member_std": 0.05482005700469017 + "member_std": 0.0631292313337326 }, "test": { - "mean_centered_corr": -0.019170163199305534, + "mean_centered_corr": -0.020325176417827606, "mean_pairwise_disagreement": NaN, - "member_std": 0.05448836833238602 + "member_std": 0.061680618673563004 } }, "chunk_size": null, diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml index 094fb6a0f..25ebc2e28 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml @@ -33,7 +33,7 @@ d_embedding = 32 n_bins = 81 [cf_fisd] -lambda = 0.05 +lambda = 0.005 variant = "raw" dataset_name = "delivery-eta" teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" @@ -42,3 +42,110 @@ teacher_names = [ "lgbm", "cat", ] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2/report.json index 6851f0002..a78e05ecf 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2/report.json @@ -38,7 +38,7 @@ "n_bins": 81 }, "cf_fisd": { - "lambda": 0.05, + "lambda": 0.005, "variant": "raw", "dataset_name": "delivery-eta", "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", @@ -46,7 +46,114 @@ "xgb", "lgbm", "cat" - ] + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } } }, "n_parameters": 6761248, @@ -59,7 +166,7 @@ } }, "cf_fisd": { - "lambda": 0.05, + "lambda": 0.005, "variant": "raw", "teacher_names": [ "xgb", @@ -78,9 +185,41 @@ 7, 8, 9, - 10 + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 ], "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, 11, 12, 13, @@ -91,9 +230,41 @@ 18, 19, 20, - 21 + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 ], "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, 22, 23, 24, @@ -329,7 +500,9 @@ 2 ], "n_features": 220, - "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" }, "epoch_size": 273, "ncl": { @@ -401,43 +574,43 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 1365, + "best_step": 1638, "metrics": { "train": { - "rmse": 0.514369679188237, - "mae": 0.4027121365070343, - "r2": 0.5040967464447021, - "score": -0.514369679188237 + "rmse": 0.5116329266744724, + "mae": 0.4038048982620239, + "r2": 0.5093597173690796, + "score": -0.5116329266744724 }, "val": { - "rmse": 0.5569470014167281, - "mae": 0.43902793526649475, - "r2": 0.4150676727294922, - "score": -0.5569470014167281 + "rmse": 0.5557493871736087, + "mae": 0.4417022466659546, + "r2": 0.4175804853439331, + "score": -0.5557493871736087 }, "test": { - "rmse": 0.5474489469980929, - "mae": 0.43932023644447327, - "r2": 0.4415346384048462, - "score": -0.5474489469980929 + "rmse": 0.5507763801521746, + "mae": 0.44514498114585876, + "r2": 0.43472522497177124, + "score": -0.5507763801521746 } }, - "time": "0:07:03.426340", + "time": "0:07:04.841597", "diversity": { "train": { - "mean_centered_corr": -0.018950974568724632, + "mean_centered_corr": -0.0205259807407856, "mean_pairwise_disagreement": NaN, - "member_std": 0.060403864830732346 + "member_std": 0.06119334325194359 }, "val": { - "mean_centered_corr": -0.019536394625902176, + "mean_centered_corr": -0.020698808133602142, "mean_pairwise_disagreement": NaN, - "member_std": 0.06044872850179672 + "member_std": 0.06522355228662491 }, "test": { - "mean_centered_corr": -0.01978791505098343, + "mean_centered_corr": -0.020667163655161858, "mean_pairwise_disagreement": NaN, - "member_std": 0.06026185676455498 + "member_std": 0.06273415684700012 } }, "chunk_size": null, diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0/report.json index e627e5168..da3358e87 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/0/report.json @@ -38,7 +38,7 @@ "n_bins": 81 }, "cf_fisd": { - "lambda": 0.05, + "lambda": 0.005, "variant": "raw", "dataset_name": "delivery-eta", "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", @@ -46,7 +46,114 @@ "xgb", "lgbm", "cat" - ] + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } } }, "n_parameters": 6761248, @@ -59,7 +166,7 @@ } }, "cf_fisd": { - "lambda": 0.05, + "lambda": 0.005, "variant": "raw", "teacher_names": [ "xgb", @@ -78,9 +185,41 @@ 7, 8, 9, - 10 + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 ], "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, 11, 12, 13, @@ -91,9 +230,41 @@ 18, 19, 20, - 21 + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 ], "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, 22, 23, 24, @@ -329,7 +500,9 @@ 2 ], "n_features": 220, - "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" }, "epoch_size": 273, "ncl": { @@ -404,54 +577,59 @@ "best_step": 2184, "metrics": { "train": { - "rmse": 0.5096923366202544, - "mae": 0.4003257751464844, - "r2": 0.5130745768547058, - "score": -0.5096923366202544 + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 }, "val": { - "rmse": 0.5556691040453957, - "mae": 0.4418928027153015, - "r2": 0.4177488088607788, - "score": -0.5556691040453957 + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 }, "test": { - "rmse": 0.5521822756750591, - "mae": 0.44639116525650024, - "r2": 0.43183577060699463, - "score": -0.5521822756750591 + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 } }, - "time": "0:08:00.261340", + "time": "0:07:43.023818", "diversity": { "train": { - "mean_centered_corr": -0.01827983371913433, + "mean_centered_corr": -0.020328260958194733, "mean_pairwise_disagreement": NaN, - "member_std": 0.0643538236618042 + "member_std": 0.06901570409536362 }, "val": { - "mean_centered_corr": -0.018152762204408646, + "mean_centered_corr": -0.020442906767129898, "mean_pairwise_disagreement": NaN, - "member_std": 0.06669643521308899 + "member_std": 0.07035460323095322 }, "test": { - "mean_centered_corr": -0.018407996743917465, + "mean_centered_corr": -0.020237648859620094, "mean_pairwise_disagreement": NaN, - "member_std": 0.06605677306652069 + "member_std": 0.06958991289138794 } }, "chunk_size": null, "eval_batch_size": 16384, "heads": [ - 30, - 24, + 13, + 6, + 10, + 14, + 26, 27, + 18, + 25, + 6, 15, - 4, - 21, - 24, - 1, - 27 + 8, + 17, + 10, + 26 ], - "head_selection_time": "0:00:00.261765" + "head_selection_time": "0:00:00.432213" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1/report.json index 49059cfd7..24cd05329 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/1/report.json @@ -38,7 +38,7 @@ "n_bins": 81 }, "cf_fisd": { - "lambda": 0.05, + "lambda": 0.005, "variant": "raw", "dataset_name": "delivery-eta", "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", @@ -46,7 +46,114 @@ "xgb", "lgbm", "cat" - ] + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } } }, "n_parameters": 6761248, @@ -59,7 +166,7 @@ } }, "cf_fisd": { - "lambda": 0.05, + "lambda": 0.005, "variant": "raw", "teacher_names": [ "xgb", @@ -78,9 +185,41 @@ 7, 8, 9, - 10 + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 ], "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, 11, 12, 13, @@ -91,9 +230,41 @@ 18, 19, 20, - 21 + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 ], "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, 22, 23, 24, @@ -329,7 +500,9 @@ 2 ], "n_features": 220, - "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" }, "epoch_size": 273, "ncl": { @@ -401,58 +574,57 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 1365, + "best_step": 1638, "metrics": { "train": { - "rmse": 0.51574838491419, - "mae": 0.4075404703617096, - "r2": 0.5014347434043884, - "score": -0.51574838491419 + "rmse": 0.5108096659852551, + "mae": 0.4014206528663635, + "r2": 0.5109373331069946, + "score": -0.5108096659852551 }, "val": { - "rmse": 0.5559175580175352, - "mae": 0.4421222507953644, - "r2": 0.41722798347473145, - "score": -0.5559175580175352 + "rmse": 0.5553836483690917, + "mae": 0.440256267786026, + "r2": 0.4183468818664551, + "score": -0.5553836483690917 }, "test": { - "rmse": 0.5539339615490284, - "mae": 0.4493137001991272, - "r2": 0.4282252788543701, - "score": -0.5539339615490284 + "rmse": 0.5515298017800222, + "mae": 0.4441634714603424, + "r2": 0.433177649974823, + "score": -0.5515298017800222 } }, - "time": "0:07:03.676796", + "time": "0:07:06.135890", "diversity": { "train": { - "mean_centered_corr": -0.0186633188277483, + "mean_centered_corr": -0.02029341645538807, "mean_pairwise_disagreement": NaN, - "member_std": 0.056279826909303665 + "member_std": 0.06168676167726517 }, "val": { - "mean_centered_corr": -0.018989257514476776, + "mean_centered_corr": -0.020232515409588814, "mean_pairwise_disagreement": NaN, - "member_std": 0.05482005700469017 + "member_std": 0.0631292313337326 }, "test": { - "mean_centered_corr": -0.019170163199305534, + "mean_centered_corr": -0.020325176417827606, "mean_pairwise_disagreement": NaN, - "member_std": 0.05448836833238602 + "member_std": 0.061680618673563004 } }, "chunk_size": null, "eval_batch_size": 16384, "heads": [ - 30, 22, - 18, - 25, - 21, + 30, 29, - 13, + 19, + 11, 18, - 26, - 22 + 30, + 4, + 16 ], - "head_selection_time": "0:00:00.286959" + "head_selection_time": "0:00:00.266709" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2/report.json index f14d6cdeb..29d133635 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_cf_fisd_only-greedy-heads-evaluation/2/report.json @@ -38,7 +38,7 @@ "n_bins": 81 }, "cf_fisd": { - "lambda": 0.05, + "lambda": 0.005, "variant": "raw", "dataset_name": "delivery-eta", "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", @@ -46,7 +46,114 @@ "xgb", "lgbm", "cat" - ] + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } } }, "n_parameters": 6761248, @@ -59,7 +166,7 @@ } }, "cf_fisd": { - "lambda": 0.05, + "lambda": 0.005, "variant": "raw", "teacher_names": [ "xgb", @@ -78,9 +185,41 @@ 7, 8, 9, - 10 + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 ], "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, 11, 12, 13, @@ -91,9 +230,41 @@ 18, 19, 20, - 21 + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 ], "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, 22, 23, 24, @@ -329,7 +500,9 @@ 2 ], "n_features": 220, - "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" }, "epoch_size": 273, "ncl": { @@ -401,57 +574,57 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 1365, + "best_step": 1638, "metrics": { "train": { - "rmse": 0.5154122867516343, - "mae": 0.4064919650554657, - "r2": 0.5020843148231506, - "score": -0.5154122867516343 + "rmse": 0.5120909698461144, + "mae": 0.4043644666671753, + "r2": 0.5084807872772217, + "score": -0.5120909698461144 }, "val": { - "rmse": 0.5559178796729317, - "mae": 0.4410634934902191, - "r2": 0.4172273278236389, - "score": -0.5559178796729317 + "rmse": 0.5554437183690072, + "mae": 0.4415339529514313, + "r2": 0.41822099685668945, + "score": -0.5554437183690072 }, "test": { - "rmse": 0.551186027318127, - "mae": 0.44573864340782166, - "r2": 0.4338840842247009, - "score": -0.551186027318127 + "rmse": 0.5516331354574737, + "mae": 0.44613465666770935, + "r2": 0.4329652190208435, + "score": -0.5516331354574737 } }, - "time": "0:07:03.426340", + "time": "0:07:04.841597", "diversity": { "train": { - "mean_centered_corr": -0.018950974568724632, + "mean_centered_corr": -0.0205259807407856, "mean_pairwise_disagreement": NaN, - "member_std": 0.060403864830732346 + "member_std": 0.06119334325194359 }, "val": { - "mean_centered_corr": -0.019536394625902176, + "mean_centered_corr": -0.020698808133602142, "mean_pairwise_disagreement": NaN, - "member_std": 0.06044872850179672 + "member_std": 0.06522355228662491 }, "test": { - "mean_centered_corr": -0.01978791505098343, + "mean_centered_corr": -0.020667163655161858, "mean_pairwise_disagreement": NaN, - "member_std": 0.06026185676455498 + "member_std": 0.06273415684700012 } }, "chunk_size": null, "eval_batch_size": 16384, "heads": [ - 24, - 29, - 13, - 15, - 29, - 25, + 8, 31, + 22, 14, - 3 + 12, + 0, + 16, + 27, + 13 ], - "head_selection_time": "0:00:00.257968" + "head_selection_time": "0:00:00.259040" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0/report.json index 97ec96d27..759452322 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/0/report.json @@ -11,18 +11,8 @@ "amp": true, "seed": 0, "head_selection": true, - "rla_adapter_lr_multiplier": 1.0, + "rla_adapter_lr_multiplier": 0.5, "rla_extra_paths_freeze_fraction": 0.0, - "use_esam": true, - "esam_rho": 0.005, - "esam_eps": 1e-12, - "esam_adapter_only": true, - "esam_memberwise": true, - "esam_warmup_epochs": 0, - "esam_start_epoch": 0, - "esam_end_epoch": -1, - "esam_log_diagnostics": false, - "esam_diagnostics_every": 100, "data": { "cache": true, "path": "data/delivery-eta", @@ -53,6 +43,124 @@ }, "bins": { "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } } }, "n_parameters": 7647904, @@ -65,7 +173,343 @@ } }, "cf_fisd": { - "lambda": 0.0 + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" }, "epoch_size": 273, "ncl": { @@ -75,15 +519,15 @@ "ncl_space": "logits" }, "esam": { - "use_esam": true, - "esam_rho": 0.005, + "use_esam": false, + "esam_rho": 0.01, "esam_eps": 1e-12, "esam_adapter_only": true, "esam_memberwise": true, "esam_warmup_epochs": 0, "esam_start_epoch": 0, "esam_end_epoch": -1, - "esam_log_diagnostics": false, + "esam_log_diagnostics": true, "esam_diagnostics_every": 100, "adapter_parameter_names": [ { @@ -133,57 +577,57 @@ ] } ], - "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "diagnostics_path": null, "n_diagnostics_records": 0 }, "rla_training": { - "adapter_lr_multiplier": 1.0, + "adapter_lr_multiplier": 0.5, "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, "best_step": 1365, "metrics": { "train": { - "rmse": 0.5197141511819273, - "mae": 0.41487401723861694, - "r2": 0.49373799562454224, - "score": -0.5197141511819273 + "rmse": 0.5175918577279733, + "mae": 0.40775081515312195, + "r2": 0.4978642463684082, + "score": -0.5175918577279733 }, "val": { - "rmse": 0.5572907783524989, - "mae": 0.4426497519016266, - "r2": 0.41434532403945923, - "score": -0.5572907783524989 + "rmse": 0.5572652691682626, + "mae": 0.44076380133628845, + "r2": 0.4143989086151123, + "score": -0.5572652691682626 }, "test": { - "rmse": 0.5545303498210034, - "mae": 0.44824421405792236, - "r2": 0.42699337005615234, - "score": -0.5545303498210034 + "rmse": 0.5524207799310179, + "mae": 0.4452172815799713, + "r2": 0.4313448667526245, + "score": -0.5524207799310179 } }, - "time": "0:25:14.112066", + "time": "0:15:04.980270", "diversity": { "train": { - "mean_centered_corr": -0.020538095384836197, + "mean_centered_corr": -0.02016017585992813, "mean_pairwise_disagreement": NaN, - "member_std": 0.056509166955947876 + "member_std": 0.06105273962020874 }, "val": { - "mean_centered_corr": -0.020886661484837532, + "mean_centered_corr": -0.01993357017636299, "mean_pairwise_disagreement": NaN, - "member_std": 0.058363690972328186 + "member_std": 0.06221357360482216 }, "test": { - "mean_centered_corr": -0.020847400650382042, + "mean_centered_corr": -0.020221160724759102, "mean_pairwise_disagreement": NaN, - "member_std": 0.057258859276771545 + "member_std": 0.06099524348974228 } }, "chunk_size": null, - "eval_batch_size": 4096, + "eval_batch_size": 8192, "heads": [ - 29 + 11 ], - "head_selection_time": "0:00:00.023366" + "head_selection_time": "0:00:00.023383" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1/report.json index 5e5e30e0d..01eb91f6a 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/1/report.json @@ -11,18 +11,8 @@ "amp": true, "seed": 1, "head_selection": true, - "rla_adapter_lr_multiplier": 1.0, + "rla_adapter_lr_multiplier": 0.5, "rla_extra_paths_freeze_fraction": 0.0, - "use_esam": true, - "esam_rho": 0.005, - "esam_eps": 1e-12, - "esam_adapter_only": true, - "esam_memberwise": true, - "esam_warmup_epochs": 0, - "esam_start_epoch": 0, - "esam_end_epoch": -1, - "esam_log_diagnostics": false, - "esam_diagnostics_every": 100, "data": { "cache": true, "path": "data/delivery-eta", @@ -53,6 +43,124 @@ }, "bins": { "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } } }, "n_parameters": 7647904, @@ -65,7 +173,343 @@ } }, "cf_fisd": { - "lambda": 0.0 + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" }, "epoch_size": 273, "ncl": { @@ -75,15 +519,15 @@ "ncl_space": "logits" }, "esam": { - "use_esam": true, - "esam_rho": 0.005, + "use_esam": false, + "esam_rho": 0.01, "esam_eps": 1e-12, "esam_adapter_only": true, "esam_memberwise": true, "esam_warmup_epochs": 0, "esam_start_epoch": 0, "esam_end_epoch": -1, - "esam_log_diagnostics": false, + "esam_log_diagnostics": true, "esam_diagnostics_every": 100, "adapter_parameter_names": [ { @@ -133,57 +577,57 @@ ] } ], - "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__1.jsonl", + "diagnostics_path": null, "n_diagnostics_records": 0 }, "rla_training": { - "adapter_lr_multiplier": 1.0, + "adapter_lr_multiplier": 0.5, "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 1638, + "best_step": 1365, "metrics": { "train": { - "rmse": 0.5103319318333938, - "mae": 0.3996802568435669, - "r2": 0.5118517279624939, - "score": -0.5103319318333938 + "rmse": 0.5171067304438404, + "mae": 0.40677744150161743, + "r2": 0.49880504608154297, + "score": -0.5171067304438404 }, "val": { - "rmse": 0.5571610811723605, - "mae": 0.440170556306839, - "r2": 0.4146178960800171, - "score": -0.5571610811723605 + "rmse": 0.5577909128836495, + "mae": 0.44260263442993164, + "r2": 0.41329365968704224, + "score": -0.5577909128836495 }, "test": { - "rmse": 0.5529389828325415, - "mae": 0.4438725709915161, - "r2": 0.4302774667739868, - "score": -0.5529389828325415 + "rmse": 0.5532065757037241, + "mae": 0.44593191146850586, + "r2": 0.4297259449958801, + "score": -0.5532065757037241 } }, - "time": "0:26:36.215265", + "time": "0:15:04.310558", "diversity": { "train": { - "mean_centered_corr": -0.02035202831029892, + "mean_centered_corr": -0.020146815106272697, "mean_pairwise_disagreement": NaN, - "member_std": 0.059233568608760834 + "member_std": 0.061014287173748016 }, "val": { - "mean_centered_corr": -0.020599549636244774, + "mean_centered_corr": -0.020077988505363464, "mean_pairwise_disagreement": NaN, - "member_std": 0.06324069947004318 + "member_std": 0.06329856067895889 }, "test": { - "mean_centered_corr": -0.020422711968421936, + "mean_centered_corr": -0.01972520351409912, "mean_pairwise_disagreement": NaN, - "member_std": 0.06101517006754875 + "member_std": 0.06241019815206528 } }, "chunk_size": null, - "eval_batch_size": 4096, + "eval_batch_size": 8192, "heads": [ - 22 + 5 ], - "head_selection_time": "0:00:00.023415" + "head_selection_time": "0:00:00.023906" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2/report.json index 615f58ce6..e40774ea4 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-best-head-evaluation/2/report.json @@ -11,18 +11,8 @@ "amp": true, "seed": 2, "head_selection": true, - "rla_adapter_lr_multiplier": 1.0, + "rla_adapter_lr_multiplier": 0.5, "rla_extra_paths_freeze_fraction": 0.0, - "use_esam": true, - "esam_rho": 0.005, - "esam_eps": 1e-12, - "esam_adapter_only": true, - "esam_memberwise": true, - "esam_warmup_epochs": 0, - "esam_start_epoch": 0, - "esam_end_epoch": -1, - "esam_log_diagnostics": false, - "esam_diagnostics_every": 100, "data": { "cache": true, "path": "data/delivery-eta", @@ -53,6 +43,124 @@ }, "bins": { "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } } }, "n_parameters": 7647904, @@ -65,7 +173,343 @@ } }, "cf_fisd": { - "lambda": 0.0 + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" }, "epoch_size": 273, "ncl": { @@ -75,15 +519,15 @@ "ncl_space": "logits" }, "esam": { - "use_esam": true, - "esam_rho": 0.005, + "use_esam": false, + "esam_rho": 0.01, "esam_eps": 1e-12, "esam_adapter_only": true, "esam_memberwise": true, "esam_warmup_epochs": 0, "esam_start_epoch": 0, "esam_end_epoch": -1, - "esam_log_diagnostics": false, + "esam_log_diagnostics": true, "esam_diagnostics_every": 100, "adapter_parameter_names": [ { @@ -133,57 +577,57 @@ ] } ], - "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__2.jsonl", + "diagnostics_path": null, "n_diagnostics_records": 0 }, "rla_training": { - "adapter_lr_multiplier": 1.0, + "adapter_lr_multiplier": 0.5, "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 819, + "best_step": 1911, "metrics": { "train": { - "rmse": 0.5218133353161428, - "mae": 0.4113169312477112, - "r2": 0.4896400570869446, - "score": -0.5218133353161428 + "rmse": 0.5137978750972679, + "mae": 0.4069885015487671, + "r2": 0.50519859790802, + "score": -0.5137978750972679 }, "val": { - "rmse": 0.5577944659141506, - "mae": 0.4427873492240906, - "r2": 0.4132862091064453, - "score": -0.5577944659141506 + "rmse": 0.5585470113340857, + "mae": 0.44582831859588623, + "r2": 0.4117019772529602, + "score": -0.5585470113340857 }, "test": { - "rmse": 0.5536175199956386, - "mae": 0.4474765956401825, - "r2": 0.4288783073425293, - "score": -0.5536175199956386 + "rmse": 0.5573737419749005, + "mae": 0.4522700607776642, + "r2": 0.4211021065711975, + "score": -0.5573737419749005 } }, - "time": "0:22:50.050598", + "time": "0:16:37.086669", "diversity": { "train": { - "mean_centered_corr": -0.019786350429058075, + "mean_centered_corr": -0.020242199301719666, "mean_pairwise_disagreement": NaN, - "member_std": 0.05782343074679375 + "member_std": 0.0667627602815628 }, "val": { - "mean_centered_corr": -0.018166765570640564, + "mean_centered_corr": -0.02033313177525997, "mean_pairwise_disagreement": NaN, - "member_std": 0.05960576981306076 + "member_std": 0.06825815886259079 }, "test": { - "mean_centered_corr": -0.019035130739212036, + "mean_centered_corr": -0.020218119025230408, "mean_pairwise_disagreement": NaN, - "member_std": 0.057923924177885056 + "member_std": 0.06682927906513214 } }, "chunk_size": null, - "eval_batch_size": 4096, + "eval_batch_size": 8192, "heads": [ - 31 + 5 ], - "head_selection_time": "0:00:00.023390" + "head_selection_time": "0:00:00.023427" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml index ce432034e..a8cfa8c75 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml @@ -5,18 +5,8 @@ gradient_clipping_norm = 1.0 amp = true seed = 0 head_selection = true -rla_adapter_lr_multiplier = 1.0 +rla_adapter_lr_multiplier = 0.5 rla_extra_paths_freeze_fraction = 0.0 -use_esam = true -esam_rho = 0.005 -esam_eps = 1e-12 -esam_adapter_only = true -esam_memberwise = true -esam_warmup_epochs = 0 -esam_start_epoch = 0 -esam_end_epoch = -1 -esam_log_diagnostics = false -esam_diagnostics_every = 100 [data] cache = true @@ -48,3 +38,121 @@ d_embedding = 32 [bins] n_bins = 81 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0/report.json index f51c2c83d..49ee599d9 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/0/report.json @@ -11,18 +11,8 @@ "amp": true, "seed": 0, "head_selection": true, - "rla_adapter_lr_multiplier": 1.0, + "rla_adapter_lr_multiplier": 0.5, "rla_extra_paths_freeze_fraction": 0.0, - "use_esam": true, - "esam_rho": 0.005, - "esam_eps": 1e-12, - "esam_adapter_only": true, - "esam_memberwise": true, - "esam_warmup_epochs": 0, - "esam_start_epoch": 0, - "esam_end_epoch": -1, - "esam_log_diagnostics": false, - "esam_diagnostics_every": 100, "data": { "cache": true, "path": "data/delivery-eta", @@ -53,6 +43,124 @@ }, "bins": { "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } } }, "n_parameters": 7647904, @@ -65,7 +173,343 @@ } }, "cf_fisd": { - "lambda": 0.0 + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" }, "epoch_size": 273, "ncl": { @@ -75,15 +519,15 @@ "ncl_space": "logits" }, "esam": { - "use_esam": true, - "esam_rho": 0.005, + "use_esam": false, + "esam_rho": 0.01, "esam_eps": 1e-12, "esam_adapter_only": true, "esam_memberwise": true, "esam_warmup_epochs": 0, "esam_start_epoch": 0, "esam_end_epoch": -1, - "esam_log_diagnostics": false, + "esam_log_diagnostics": true, "esam_diagnostics_every": 100, "adapter_parameter_names": [ { @@ -133,53 +577,53 @@ ] } ], - "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "diagnostics_path": null, "n_diagnostics_records": 0 }, "rla_training": { - "adapter_lr_multiplier": 1.0, + "adapter_lr_multiplier": 0.5, "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, "best_step": 1365, "metrics": { "train": { - "rmse": 0.5146695445761407, - "mae": 0.40757593512535095, - "r2": 0.5035183429718018, - "score": -0.5146695445761407 + "rmse": 0.5150130136938161, + "mae": 0.40628108382225037, + "r2": 0.5028554797172546, + "score": -0.5150130136938161 }, "val": { - "rmse": 0.5559417888696784, - "mae": 0.44435763359069824, - "r2": 0.4171772003173828, - "score": -0.5559417888696784 + "rmse": 0.5559356776481553, + "mae": 0.441616415977478, + "r2": 0.4171900153160095, + "score": -0.5559356776481553 }, "test": { - "rmse": 0.5541331785404355, - "mae": 0.44994547963142395, - "r2": 0.42781394720077515, - "score": -0.5541331785404355 + "rmse": 0.55135486033622, + "mae": 0.445721298456192, + "r2": 0.43353718519210815, + "score": -0.55135486033622 } }, - "time": "0:25:14.112066", + "time": "0:15:04.980270", "diversity": { "train": { - "mean_centered_corr": -0.020538095384836197, + "mean_centered_corr": -0.02016017585992813, "mean_pairwise_disagreement": NaN, - "member_std": 0.056509166955947876 + "member_std": 0.06105273962020874 }, "val": { - "mean_centered_corr": -0.020886661484837532, + "mean_centered_corr": -0.01993357017636299, "mean_pairwise_disagreement": NaN, - "member_std": 0.058363690972328186 + "member_std": 0.06221357360482216 }, "test": { - "mean_centered_corr": -0.020847400650382042, + "mean_centered_corr": -0.020221160724759102, "mean_pairwise_disagreement": NaN, - "member_std": 0.057258859276771545 + "member_std": 0.06099524348974228 } }, "chunk_size": null, - "eval_batch_size": 4096 + "eval_batch_size": 8192 } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml index 2d5565099..7c4ec352c 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml @@ -5,18 +5,8 @@ gradient_clipping_norm = 1.0 amp = true seed = 1 head_selection = true -rla_adapter_lr_multiplier = 1.0 +rla_adapter_lr_multiplier = 0.5 rla_extra_paths_freeze_fraction = 0.0 -use_esam = true -esam_rho = 0.005 -esam_eps = 1e-12 -esam_adapter_only = true -esam_memberwise = true -esam_warmup_epochs = 0 -esam_start_epoch = 0 -esam_end_epoch = -1 -esam_log_diagnostics = false -esam_diagnostics_every = 100 [data] cache = true @@ -48,3 +38,121 @@ d_embedding = 32 [bins] n_bins = 81 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1/report.json index d06153cf8..a02af59d6 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/1/report.json @@ -11,18 +11,8 @@ "amp": true, "seed": 1, "head_selection": true, - "rla_adapter_lr_multiplier": 1.0, + "rla_adapter_lr_multiplier": 0.5, "rla_extra_paths_freeze_fraction": 0.0, - "use_esam": true, - "esam_rho": 0.005, - "esam_eps": 1e-12, - "esam_adapter_only": true, - "esam_memberwise": true, - "esam_warmup_epochs": 0, - "esam_start_epoch": 0, - "esam_end_epoch": -1, - "esam_log_diagnostics": false, - "esam_diagnostics_every": 100, "data": { "cache": true, "path": "data/delivery-eta", @@ -53,6 +43,124 @@ }, "bins": { "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } } }, "n_parameters": 7647904, @@ -65,7 +173,343 @@ } }, "cf_fisd": { - "lambda": 0.0 + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" }, "epoch_size": 273, "ncl": { @@ -75,15 +519,15 @@ "ncl_space": "logits" }, "esam": { - "use_esam": true, - "esam_rho": 0.005, + "use_esam": false, + "esam_rho": 0.01, "esam_eps": 1e-12, "esam_adapter_only": true, "esam_memberwise": true, "esam_warmup_epochs": 0, "esam_start_epoch": 0, "esam_end_epoch": -1, - "esam_log_diagnostics": false, + "esam_log_diagnostics": true, "esam_diagnostics_every": 100, "adapter_parameter_names": [ { @@ -133,53 +577,53 @@ ] } ], - "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__1.jsonl", + "diagnostics_path": null, "n_diagnostics_records": 0 }, "rla_training": { - "adapter_lr_multiplier": 1.0, + "adapter_lr_multiplier": 0.5, "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 1638, + "best_step": 1365, "metrics": { "train": { - "rmse": 0.5087316557555162, - "mae": 0.39884668588638306, - "r2": 0.5149083733558655, - "score": -0.5087316557555162 + "rmse": 0.5138602835910225, + "mae": 0.4030267894268036, + "r2": 0.5050784349441528, + "score": -0.5138602835910225 }, "val": { - "rmse": 0.5563776030607058, - "mae": 0.43999341130256653, - "r2": 0.4162631034851074, - "score": -0.5563776030607058 + "rmse": 0.5564338969448529, + "mae": 0.43967634439468384, + "r2": 0.4161449074745178, + "score": -0.5564338969448529 }, "test": { - "rmse": 0.5504132928776152, - "mae": 0.4417083263397217, - "r2": 0.4354702830314636, - "score": -0.5504132928776152 + "rmse": 0.5491088203688107, + "mae": 0.4416921138763428, + "r2": 0.43814295530319214, + "score": -0.5491088203688107 } }, - "time": "0:26:36.215265", + "time": "0:15:04.310558", "diversity": { "train": { - "mean_centered_corr": -0.02035202831029892, + "mean_centered_corr": -0.020146815106272697, "mean_pairwise_disagreement": NaN, - "member_std": 0.059233568608760834 + "member_std": 0.061014287173748016 }, "val": { - "mean_centered_corr": -0.020599549636244774, + "mean_centered_corr": -0.020077988505363464, "mean_pairwise_disagreement": NaN, - "member_std": 0.06324069947004318 + "member_std": 0.06329856067895889 }, "test": { - "mean_centered_corr": -0.020422711968421936, + "mean_centered_corr": -0.01972520351409912, "mean_pairwise_disagreement": NaN, - "member_std": 0.06101517006754875 + "member_std": 0.06241019815206528 } }, "chunk_size": null, - "eval_batch_size": 4096 + "eval_batch_size": 8192 } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml index 3610629d9..d99f2962a 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml @@ -5,18 +5,8 @@ gradient_clipping_norm = 1.0 amp = true seed = 2 head_selection = true -rla_adapter_lr_multiplier = 1.0 +rla_adapter_lr_multiplier = 0.5 rla_extra_paths_freeze_fraction = 0.0 -use_esam = true -esam_rho = 0.005 -esam_eps = 1e-12 -esam_adapter_only = true -esam_memberwise = true -esam_warmup_epochs = 0 -esam_start_epoch = 0 -esam_end_epoch = -1 -esam_log_diagnostics = false -esam_diagnostics_every = 100 [data] cache = true @@ -48,3 +38,121 @@ d_embedding = 32 [bins] n_bins = 81 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2/report.json index 0e8fd44b8..b225106f7 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/2/report.json @@ -11,18 +11,8 @@ "amp": true, "seed": 2, "head_selection": true, - "rla_adapter_lr_multiplier": 1.0, + "rla_adapter_lr_multiplier": 0.5, "rla_extra_paths_freeze_fraction": 0.0, - "use_esam": true, - "esam_rho": 0.005, - "esam_eps": 1e-12, - "esam_adapter_only": true, - "esam_memberwise": true, - "esam_warmup_epochs": 0, - "esam_start_epoch": 0, - "esam_end_epoch": -1, - "esam_log_diagnostics": false, - "esam_diagnostics_every": 100, "data": { "cache": true, "path": "data/delivery-eta", @@ -53,6 +43,124 @@ }, "bins": { "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } } }, "n_parameters": 7647904, @@ -65,7 +173,343 @@ } }, "cf_fisd": { - "lambda": 0.0 + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" }, "epoch_size": 273, "ncl": { @@ -75,15 +519,15 @@ "ncl_space": "logits" }, "esam": { - "use_esam": true, - "esam_rho": 0.005, + "use_esam": false, + "esam_rho": 0.01, "esam_eps": 1e-12, "esam_adapter_only": true, "esam_memberwise": true, "esam_warmup_epochs": 0, "esam_start_epoch": 0, "esam_end_epoch": -1, - "esam_log_diagnostics": false, + "esam_log_diagnostics": true, "esam_diagnostics_every": 100, "adapter_parameter_names": [ { @@ -133,53 +577,53 @@ ] } ], - "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__2.jsonl", + "diagnostics_path": null, "n_diagnostics_records": 0 }, "rla_training": { - "adapter_lr_multiplier": 1.0, + "adapter_lr_multiplier": 0.5, "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 819, + "best_step": 1911, "metrics": { "train": { - "rmse": 0.518709062776439, - "mae": 0.407330721616745, - "r2": 0.49569422006607056, - "score": -0.518709062776439 + "rmse": 0.5080919523858995, + "mae": 0.3977190852165222, + "r2": 0.5161275267601013, + "score": -0.5080919523858995 }, "val": { - "rmse": 0.556831808887752, - "mae": 0.44260597229003906, - "r2": 0.4153096079826355, - "score": -0.556831808887752 + "rmse": 0.5562272527951271, + "mae": 0.43906888365745544, + "r2": 0.416578471660614, + "score": -0.5562272527951271 }, "test": { - "rmse": 0.5516551774702309, - "mae": 0.4460088610649109, - "r2": 0.432919979095459, - "score": -0.5516551774702309 + "rmse": 0.5494697322909359, + "mae": 0.44080713391304016, + "r2": 0.43740415573120117, + "score": -0.5494697322909359 } }, - "time": "0:22:50.050598", + "time": "0:16:37.086669", "diversity": { "train": { - "mean_centered_corr": -0.019786350429058075, + "mean_centered_corr": -0.020242199301719666, "mean_pairwise_disagreement": NaN, - "member_std": 0.05782343074679375 + "member_std": 0.0667627602815628 }, "val": { - "mean_centered_corr": -0.018166765570640564, + "mean_centered_corr": -0.02033313177525997, "mean_pairwise_disagreement": NaN, - "member_std": 0.05960576981306076 + "member_std": 0.06825815886259079 }, "test": { - "mean_centered_corr": -0.019035130739212036, + "mean_centered_corr": -0.020218119025230408, "mean_pairwise_disagreement": NaN, - "member_std": 0.057923924177885056 + "member_std": 0.06682927906513214 } }, "chunk_size": null, - "eval_batch_size": 4096 + "eval_batch_size": 8192 } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0/report.json index 5449bd7de..f1e9f1689 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/0/report.json @@ -11,18 +11,8 @@ "amp": true, "seed": 0, "head_selection": true, - "rla_adapter_lr_multiplier": 1.0, + "rla_adapter_lr_multiplier": 0.5, "rla_extra_paths_freeze_fraction": 0.0, - "use_esam": true, - "esam_rho": 0.005, - "esam_eps": 1e-12, - "esam_adapter_only": true, - "esam_memberwise": true, - "esam_warmup_epochs": 0, - "esam_start_epoch": 0, - "esam_end_epoch": -1, - "esam_log_diagnostics": false, - "esam_diagnostics_every": 100, "data": { "cache": true, "path": "data/delivery-eta", @@ -53,6 +43,124 @@ }, "bins": { "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } } }, "n_parameters": 7647904, @@ -65,7 +173,343 @@ } }, "cf_fisd": { - "lambda": 0.0 + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" }, "epoch_size": 273, "ncl": { @@ -75,15 +519,15 @@ "ncl_space": "logits" }, "esam": { - "use_esam": true, - "esam_rho": 0.005, + "use_esam": false, + "esam_rho": 0.01, "esam_eps": 1e-12, "esam_adapter_only": true, "esam_memberwise": true, "esam_warmup_epochs": 0, "esam_start_epoch": 0, "esam_end_epoch": -1, - "esam_log_diagnostics": false, + "esam_log_diagnostics": true, "esam_diagnostics_every": 100, "adapter_parameter_names": [ { @@ -133,64 +577,65 @@ ] } ], - "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/_esam_diagnostics/rla_esam-evaluation__0.jsonl", + "diagnostics_path": null, "n_diagnostics_records": 0 }, "rla_training": { - "adapter_lr_multiplier": 1.0, + "adapter_lr_multiplier": 0.5, "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, "best_step": 1365, "metrics": { "train": { - "rmse": 0.5146023695303711, - "mae": 0.4073488712310791, - "r2": 0.5036479234695435, - "score": -0.5146023695303711 + "rmse": 0.5148489632002486, + "mae": 0.40648993849754333, + "r2": 0.5031721591949463, + "score": -0.5148489632002486 }, "val": { - "rmse": 0.5552988041877751, - "mae": 0.44173434376716614, - "r2": 0.4185245633125305, - "score": -0.5552988041877751 + "rmse": 0.5553930389270275, + "mae": 0.4419495463371277, + "r2": 0.4183272123336792, + "score": -0.5553930389270275 }, "test": { - "rmse": 0.5525187688548042, - "mae": 0.44692304730415344, - "r2": 0.43114304542541504, - "score": -0.5525187688548042 + "rmse": 0.5519483119271499, + "mae": 0.4472074508666992, + "r2": 0.4323171377182007, + "score": -0.5519483119271499 } }, - "time": "0:25:14.112066", + "time": "0:15:04.980270", "diversity": { "train": { - "mean_centered_corr": -0.020538095384836197, + "mean_centered_corr": -0.02016017585992813, "mean_pairwise_disagreement": NaN, - "member_std": 0.056509166955947876 + "member_std": 0.06105273962020874 }, "val": { - "mean_centered_corr": -0.020886661484837532, + "mean_centered_corr": -0.01993357017636299, "mean_pairwise_disagreement": NaN, - "member_std": 0.058363690972328186 + "member_std": 0.06221357360482216 }, "test": { - "mean_centered_corr": -0.020847400650382042, + "mean_centered_corr": -0.020221160724759102, "mean_pairwise_disagreement": NaN, - "member_std": 0.057258859276771545 + "member_std": 0.06099524348974228 } }, "chunk_size": null, - "eval_batch_size": 4096, + "eval_batch_size": 8192, "heads": [ - 29, - 16, 11, 10, 26, - 8, + 21, 15, - 22 + 16, + 31, + 19, + 13 ], - "head_selection_time": "0:00:00.230227" + "head_selection_time": "0:00:00.266220" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1/report.json index 0ef3cfea2..486356000 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/1/report.json @@ -11,18 +11,8 @@ "amp": true, "seed": 1, "head_selection": true, - "rla_adapter_lr_multiplier": 1.0, + "rla_adapter_lr_multiplier": 0.5, "rla_extra_paths_freeze_fraction": 0.0, - "use_esam": true, - "esam_rho": 0.005, - "esam_eps": 1e-12, - "esam_adapter_only": true, - "esam_memberwise": true, - "esam_warmup_epochs": 0, - "esam_start_epoch": 0, - "esam_end_epoch": -1, - "esam_log_diagnostics": false, - "esam_diagnostics_every": 100, "data": { "cache": true, "path": "data/delivery-eta", @@ -53,6 +43,124 @@ }, "bins": { "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } } }, "n_parameters": 7647904, @@ -65,7 +173,343 @@ } }, "cf_fisd": { - "lambda": 0.0 + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" }, "epoch_size": 273, "ncl": { @@ -75,15 +519,15 @@ "ncl_space": "logits" }, "esam": { - "use_esam": true, - "esam_rho": 0.005, + "use_esam": false, + "esam_rho": 0.01, "esam_eps": 1e-12, "esam_adapter_only": true, "esam_memberwise": true, "esam_warmup_epochs": 0, "esam_start_epoch": 0, "esam_end_epoch": -1, - "esam_log_diagnostics": false, + "esam_log_diagnostics": true, "esam_diagnostics_every": 100, "adapter_parameter_names": [ { @@ -133,61 +577,66 @@ ] } ], - "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__1.jsonl", + "diagnostics_path": null, "n_diagnostics_records": 0 }, "rla_training": { - "adapter_lr_multiplier": 1.0, + "adapter_lr_multiplier": 0.5, "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 1638, + "best_step": 1365, "metrics": { "train": { - "rmse": 0.5090920101016853, - "mae": 0.39969104528427124, - "r2": 0.5142209529876709, - "score": -0.5090920101016853 + "rmse": 0.5146326283410019, + "mae": 0.4055343568325043, + "r2": 0.5035895109176636, + "score": -0.5146326283410019 }, "val": { - "rmse": 0.5558474595058753, - "mae": 0.44084590673446655, - "r2": 0.41737496852874756, - "score": -0.5558474595058753 + "rmse": 0.5557605143473513, + "mae": 0.4405376613140106, + "r2": 0.4175572395324707, + "score": -0.5557605143473513 }, "test": { - "rmse": 0.5521241448870569, - "mae": 0.44504550099372864, - "r2": 0.43195533752441406, - "score": -0.5521241448870569 + "rmse": 0.5514412571081766, + "mae": 0.4451921284198761, + "r2": 0.43335968255996704, + "score": -0.5514412571081766 } }, - "time": "0:26:36.215265", + "time": "0:15:04.310558", "diversity": { "train": { - "mean_centered_corr": -0.02035202831029892, + "mean_centered_corr": -0.020146815106272697, "mean_pairwise_disagreement": NaN, - "member_std": 0.059233568608760834 + "member_std": 0.061014287173748016 }, "val": { - "mean_centered_corr": -0.020599549636244774, + "mean_centered_corr": -0.020077988505363464, "mean_pairwise_disagreement": NaN, - "member_std": 0.06324069947004318 + "member_std": 0.06329856067895889 }, "test": { - "mean_centered_corr": -0.020422711968421936, + "mean_centered_corr": -0.01972520351409912, "mean_pairwise_disagreement": NaN, - "member_std": 0.06101517006754875 + "member_std": 0.06241019815206528 } }, "chunk_size": null, - "eval_batch_size": 4096, + "eval_batch_size": 8192, "heads": [ + 5, + 12, + 30, 22, 29, - 30, + 18, + 31, + 25, 4, - 13 + 14 ], - "head_selection_time": "0:00:00.148296" + "head_selection_time": "0:00:00.299075" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2/report.json index 4dadccf56..5afeef8d4 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-greedy-heads-evaluation/2/report.json @@ -11,18 +11,8 @@ "amp": true, "seed": 2, "head_selection": true, - "rla_adapter_lr_multiplier": 1.0, + "rla_adapter_lr_multiplier": 0.5, "rla_extra_paths_freeze_fraction": 0.0, - "use_esam": true, - "esam_rho": 0.005, - "esam_eps": 1e-12, - "esam_adapter_only": true, - "esam_memberwise": true, - "esam_warmup_epochs": 0, - "esam_start_epoch": 0, - "esam_end_epoch": -1, - "esam_log_diagnostics": false, - "esam_diagnostics_every": 100, "data": { "cache": true, "path": "data/delivery-eta", @@ -53,6 +43,124 @@ }, "bins": { "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } } }, "n_parameters": 7647904, @@ -65,7 +173,343 @@ } }, "cf_fisd": { - "lambda": 0.0 + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" }, "epoch_size": 273, "ncl": { @@ -75,15 +519,15 @@ "ncl_space": "logits" }, "esam": { - "use_esam": true, - "esam_rho": 0.005, + "use_esam": false, + "esam_rho": 0.01, "esam_eps": 1e-12, "esam_adapter_only": true, "esam_memberwise": true, "esam_warmup_epochs": 0, "esam_start_epoch": 0, "esam_end_epoch": -1, - "esam_log_diagnostics": false, + "esam_log_diagnostics": true, "esam_diagnostics_every": 100, "adapter_parameter_names": [ { @@ -133,68 +577,71 @@ ] } ], - "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_combined-evaluation/_esam_diagnostics/best_combined-evaluation__2.jsonl", + "diagnostics_path": null, "n_diagnostics_records": 0 }, "rla_training": { - "adapter_lr_multiplier": 1.0, + "adapter_lr_multiplier": 0.5, "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 819, + "best_step": 1911, "metrics": { "train": { - "rmse": 0.5186568050088503, - "mae": 0.40700122714042664, - "r2": 0.4957958459854126, - "score": -0.5186568050088503 + "rmse": 0.5091281866287168, + "mae": 0.40107253193855286, + "r2": 0.5141518115997314, + "score": -0.5091281866287168 }, "val": { - "rmse": 0.5564955940706238, - "mae": 0.4424670338630676, - "r2": 0.41601550579071045, - "score": -0.5564955940706238 + "rmse": 0.5554844141815782, + "mae": 0.4408845603466034, + "r2": 0.41813576221466064, + "score": -0.5554844141815782 }, "test": { - "rmse": 0.5523659114897095, - "mae": 0.446776419878006, - "r2": 0.4314578175544739, - "score": -0.5523659114897095 + "rmse": 0.5520847398512095, + "mae": 0.44570744037628174, + "r2": 0.4320363998413086, + "score": -0.5520847398512095 } }, - "time": "0:22:50.050598", + "time": "0:16:37.086669", "diversity": { "train": { - "mean_centered_corr": -0.019786350429058075, + "mean_centered_corr": -0.020242199301719666, "mean_pairwise_disagreement": NaN, - "member_std": 0.05782343074679375 + "member_std": 0.0667627602815628 }, "val": { - "mean_centered_corr": -0.018166765570640564, + "mean_centered_corr": -0.02033313177525997, "mean_pairwise_disagreement": NaN, - "member_std": 0.05960576981306076 + "member_std": 0.06825815886259079 }, "test": { - "mean_centered_corr": -0.019035130739212036, + "mean_centered_corr": -0.020218119025230408, "mean_pairwise_disagreement": NaN, - "member_std": 0.057923924177885056 + "member_std": 0.06682927906513214 } }, "chunk_size": null, - "eval_batch_size": 4096, + "eval_batch_size": 8192, "heads": [ - 31, - 6, - 25, - 8, - 19, - 22, - 2, - 12, - 18, - 29, 5, - 13 + 14, + 23, + 13, + 0, + 24, + 7, + 16, + 29, + 8, + 6, + 14, + 27, + 7, + 18 ], - "head_selection_time": "0:00:00.360643" + "head_selection_time": "0:00:00.472757" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/0/report.json index 6e8396567..2b84e1465 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/0/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/0/report.json @@ -12,12 +12,12 @@ "seed": 0, "head_selection": true, "use_esam": true, - "esam_rho": 0.005, + "esam_rho": 0.001, "esam_eps": 1e-12, - "esam_adapter_only": true, + "esam_adapter_only": false, "esam_memberwise": true, "esam_warmup_epochs": 0, - "esam_start_epoch": 0, + "esam_start_epoch": 8, "esam_end_epoch": -1, "esam_log_diagnostics": false, "esam_diagnostics_every": 100, @@ -69,16 +69,45 @@ }, "esam": { "use_esam": true, - "esam_rho": 0.005, + "esam_rho": 0.001, "esam_eps": 1e-12, - "esam_adapter_only": true, + "esam_adapter_only": false, "esam_memberwise": true, "esam_warmup_epochs": 0, - "esam_start_epoch": 0, + "esam_start_epoch": 8, "esam_end_epoch": -1, "esam_log_diagnostics": false, "esam_diagnostics_every": 100, "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, { "name": "backbone.blocks.0.0.r", "shape": [ @@ -100,6 +129,13 @@ 752 ] }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, { "name": "backbone.blocks.1.0.r", "shape": [ @@ -120,9 +156,24 @@ 32, 752 ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] } ], - "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__0.jsonl", "n_diagnostics_records": 0 }, "rla_training": { @@ -130,49 +181,49 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 2184, + "best_step": 2730, "metrics": { "train": { - "rmse": 0.5123514530013489, - "mae": 0.4078504741191864, - "r2": 0.5079806447029114, - "score": -0.5123514530013489 + "rmse": 0.5056157353876229, + "mae": 0.3998609483242035, + "r2": 0.5208324193954468, + "score": -0.5056157353876229 }, "val": { - "rmse": 0.5582863288518123, - "mae": 0.4448302686214447, - "r2": 0.41225099563598633, - "score": -0.5582863288518123 + "rmse": 0.5596782586864594, + "mae": 0.44516822695732117, + "r2": 0.4093165993690491, + "score": -0.5596782586864594 }, "test": { - "rmse": 0.5560390501031047, - "mae": 0.4517032206058502, - "r2": 0.4238712191581726, - "score": -0.5560390501031047 + "rmse": 0.5565908577070521, + "mae": 0.45004433393478394, + "r2": 0.42272716760635376, + "score": -0.5565908577070521 } }, - "time": "0:12:51.556071", + "time": "0:13:12.089007", "diversity": { "train": { - "mean_centered_corr": -0.02027146704494953, + "mean_centered_corr": -0.020204994827508926, "mean_pairwise_disagreement": NaN, - "member_std": 0.06810352951288223 + "member_std": 0.07431785017251968 }, "val": { - "mean_centered_corr": -0.02014581859111786, + "mean_centered_corr": -0.020286401733756065, "mean_pairwise_disagreement": NaN, - "member_std": 0.07032498717308044 + "member_std": 0.07910650968551636 }, "test": { - "mean_centered_corr": -0.02043480984866619, + "mean_centered_corr": -0.02015332505106926, "mean_pairwise_disagreement": NaN, - "member_std": 0.0694202110171318 + "member_std": 0.07623301446437836 } }, "chunk_size": null, - "eval_batch_size": 8192, + "eval_batch_size": 16384, "heads": [ - 13 + 24 ], - "head_selection_time": "0:00:00.023277" + "head_selection_time": "0:00:00.024366" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/1/report.json index 338753c41..50d3cc1a2 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/1/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/1/report.json @@ -12,12 +12,12 @@ "seed": 1, "head_selection": true, "use_esam": true, - "esam_rho": 0.005, + "esam_rho": 0.001, "esam_eps": 1e-12, - "esam_adapter_only": true, + "esam_adapter_only": false, "esam_memberwise": true, "esam_warmup_epochs": 0, - "esam_start_epoch": 0, + "esam_start_epoch": 8, "esam_end_epoch": -1, "esam_log_diagnostics": false, "esam_diagnostics_every": 100, @@ -69,16 +69,45 @@ }, "esam": { "use_esam": true, - "esam_rho": 0.005, + "esam_rho": 0.001, "esam_eps": 1e-12, - "esam_adapter_only": true, + "esam_adapter_only": false, "esam_memberwise": true, "esam_warmup_epochs": 0, - "esam_start_epoch": 0, + "esam_start_epoch": 8, "esam_end_epoch": -1, "esam_log_diagnostics": false, "esam_diagnostics_every": 100, "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, { "name": "backbone.blocks.0.0.r", "shape": [ @@ -100,6 +129,13 @@ 752 ] }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, { "name": "backbone.blocks.1.0.r", "shape": [ @@ -120,6 +156,21 @@ 32, 752 ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] } ], "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", @@ -133,46 +184,46 @@ "best_step": 1638, "metrics": { "train": { - "rmse": 0.5159431112020112, - "mae": 0.4095075726509094, - "r2": 0.5010582208633423, - "score": -0.5159431112020112 + "rmse": 0.5157048423679549, + "mae": 0.408806711435318, + "r2": 0.5015189051628113, + "score": -0.5157048423679549 }, "val": { - "rmse": 0.5579506166150647, - "mae": 0.4473210573196411, - "r2": 0.41295766830444336, - "score": -0.5579506166150647 + "rmse": 0.5578575885405295, + "mae": 0.44588109850883484, + "r2": 0.41315340995788574, + "score": -0.5578575885405295 }, "test": { - "rmse": 0.5600144253303337, - "mae": 0.4565167725086212, - "r2": 0.41560375690460205, - "score": -0.5600144253303337 + "rmse": 0.5568376961917461, + "mae": 0.4522704780101776, + "r2": 0.4222150444984436, + "score": -0.5568376961917461 } }, - "time": "0:11:50.725269", + "time": "0:10:49.864209", "diversity": { "train": { - "mean_centered_corr": -0.020498434081673622, + "mean_centered_corr": -0.02029341645538807, "mean_pairwise_disagreement": NaN, - "member_std": 0.06282930076122284 + "member_std": 0.06168676167726517 }, "val": { - "mean_centered_corr": -0.020406439900398254, + "mean_centered_corr": -0.020232515409588814, "mean_pairwise_disagreement": NaN, - "member_std": 0.06703978031873703 + "member_std": 0.0631292313337326 }, "test": { - "mean_centered_corr": -0.020546942949295044, + "mean_centered_corr": -0.020325176417827606, "mean_pairwise_disagreement": NaN, - "member_std": 0.06463686376810074 + "member_std": 0.061680618673563004 } }, "chunk_size": null, - "eval_batch_size": 8192, + "eval_batch_size": 16384, "heads": [ 22 ], - "head_selection_time": "0:00:00.023511" + "head_selection_time": "0:00:00.024983" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/2/report.json index 6c097eae7..af23e2c48 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/2/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-best-head-evaluation/2/report.json @@ -12,12 +12,12 @@ "seed": 2, "head_selection": true, "use_esam": true, - "esam_rho": 0.005, + "esam_rho": 0.001, "esam_eps": 1e-12, - "esam_adapter_only": true, + "esam_adapter_only": false, "esam_memberwise": true, "esam_warmup_epochs": 0, - "esam_start_epoch": 0, + "esam_start_epoch": 8, "esam_end_epoch": -1, "esam_log_diagnostics": false, "esam_diagnostics_every": 100, @@ -69,16 +69,45 @@ }, "esam": { "use_esam": true, - "esam_rho": 0.005, + "esam_rho": 0.001, "esam_eps": 1e-12, - "esam_adapter_only": true, + "esam_adapter_only": false, "esam_memberwise": true, "esam_warmup_epochs": 0, - "esam_start_epoch": 0, + "esam_start_epoch": 8, "esam_end_epoch": -1, "esam_log_diagnostics": false, "esam_diagnostics_every": 100, "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, { "name": "backbone.blocks.0.0.r", "shape": [ @@ -100,6 +129,13 @@ 752 ] }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, { "name": "backbone.blocks.1.0.r", "shape": [ @@ -120,6 +156,21 @@ 32, 752 ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] } ], "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", @@ -133,46 +184,46 @@ "best_step": 1638, "metrics": { "train": { - "rmse": 0.5152948652017916, - "mae": 0.40833914279937744, - "r2": 0.5023112297058105, - "score": -0.5152948652017916 + "rmse": 0.5148308736438392, + "mae": 0.40706372261047363, + "r2": 0.5032070279121399, + "score": -0.5148308736438392 }, "val": { - "rmse": 0.5578059530007539, - "mae": 0.4471565783023834, - "r2": 0.4132620096206665, - "score": -0.5578059530007539 + "rmse": 0.5576450053690213, + "mae": 0.4445996880531311, + "r2": 0.4136006236076355, + "score": -0.5576450053690213 }, "test": { - "rmse": 0.5592548752846349, - "mae": 0.45557308197021484, - "r2": 0.4171879291534424, - "score": -0.5592548752846349 + "rmse": 0.5574656479348947, + "mae": 0.45258787274360657, + "r2": 0.42091113328933716, + "score": -0.5574656479348947 } }, - "time": "0:11:47.829319", + "time": "0:10:43.789154", "diversity": { "train": { - "mean_centered_corr": -0.020578308030962944, + "mean_centered_corr": -0.0205259807407856, "mean_pairwise_disagreement": NaN, - "member_std": 0.06090719997882843 + "member_std": 0.06119334325194359 }, "val": { - "mean_centered_corr": -0.020749453455209732, + "mean_centered_corr": -0.020698808133602142, "mean_pairwise_disagreement": NaN, - "member_std": 0.06413886696100235 + "member_std": 0.06522355228662491 }, "test": { - "mean_centered_corr": -0.020869312807917595, + "mean_centered_corr": -0.020667163655161858, "mean_pairwise_disagreement": NaN, - "member_std": 0.061883412301540375 + "member_std": 0.06273415684700012 } }, "chunk_size": null, - "eval_batch_size": 8192, + "eval_batch_size": 16384, "heads": [ - 28 + 8 ], - "head_selection_time": "0:00:00.023299" + "head_selection_time": "0:00:00.023410" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml index 4b93a10d8..474196b1e 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml @@ -6,12 +6,12 @@ amp = true seed = 0 head_selection = true use_esam = true -esam_rho = 0.005 +esam_rho = 0.001 esam_eps = 1e-12 -esam_adapter_only = true +esam_adapter_only = false esam_memberwise = true esam_warmup_epochs = 0 -esam_start_epoch = 0 +esam_start_epoch = 8 esam_end_epoch = -1 esam_log_diagnostics = false esam_diagnostics_every = 100 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0/report.json index 239380f66..f91f1eae8 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/0/report.json @@ -12,12 +12,12 @@ "seed": 0, "head_selection": true, "use_esam": true, - "esam_rho": 0.005, + "esam_rho": 0.001, "esam_eps": 1e-12, - "esam_adapter_only": true, + "esam_adapter_only": false, "esam_memberwise": true, "esam_warmup_epochs": 0, - "esam_start_epoch": 0, + "esam_start_epoch": 8, "esam_end_epoch": -1, "esam_log_diagnostics": false, "esam_diagnostics_every": 100, @@ -69,16 +69,45 @@ }, "esam": { "use_esam": true, - "esam_rho": 0.005, + "esam_rho": 0.001, "esam_eps": 1e-12, - "esam_adapter_only": true, + "esam_adapter_only": false, "esam_memberwise": true, "esam_warmup_epochs": 0, - "esam_start_epoch": 0, + "esam_start_epoch": 8, "esam_end_epoch": -1, "esam_log_diagnostics": false, "esam_diagnostics_every": 100, "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, { "name": "backbone.blocks.0.0.r", "shape": [ @@ -100,6 +129,13 @@ 752 ] }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, { "name": "backbone.blocks.1.0.r", "shape": [ @@ -120,9 +156,24 @@ 32, 752 ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] } ], - "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__0.jsonl", "n_diagnostics_records": 0 }, "rla_training": { @@ -130,45 +181,45 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 2184, + "best_step": 2730, "metrics": { "train": { - "rmse": 0.504551673775848, - "mae": 0.39544814825057983, - "r2": 0.5228471159934998, - "score": -0.504551673775848 + "rmse": 0.4975037099949126, + "mae": 0.39285650849342346, + "r2": 0.5360844135284424, + "score": -0.4975037099949126 }, "val": { - "rmse": 0.5556360918188752, - "mae": 0.439733624458313, - "r2": 0.41781800985336304, - "score": -0.5556360918188752 + "rmse": 0.5558269510040955, + "mae": 0.44211816787719727, + "r2": 0.4174179434776306, + "score": -0.5558269510040955 }, "test": { - "rmse": 0.5494649592968895, - "mae": 0.4416595995426178, - "r2": 0.43741387128829956, - "score": -0.5494649592968895 + "rmse": 0.5527269612276788, + "mae": 0.44693246483802795, + "r2": 0.4307142496109009, + "score": -0.5527269612276788 } }, - "time": "0:12:51.556071", + "time": "0:13:12.089007", "diversity": { "train": { - "mean_centered_corr": -0.02027146704494953, + "mean_centered_corr": -0.020204994827508926, "mean_pairwise_disagreement": NaN, - "member_std": 0.06810352951288223 + "member_std": 0.07431785017251968 }, "val": { - "mean_centered_corr": -0.02014581859111786, + "mean_centered_corr": -0.020286401733756065, "mean_pairwise_disagreement": NaN, - "member_std": 0.07032498717308044 + "member_std": 0.07910650968551636 }, "test": { - "mean_centered_corr": -0.02043480984866619, + "mean_centered_corr": -0.02015332505106926, "mean_pairwise_disagreement": NaN, - "member_std": 0.0694202110171318 + "member_std": 0.07623301446437836 } }, "chunk_size": null, - "eval_batch_size": 8192 + "eval_batch_size": 16384 } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml index 41dae5a50..d8c270d0c 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml @@ -6,12 +6,12 @@ amp = true seed = 1 head_selection = true use_esam = true -esam_rho = 0.005 +esam_rho = 0.001 esam_eps = 1e-12 -esam_adapter_only = true +esam_adapter_only = false esam_memberwise = true esam_warmup_epochs = 0 -esam_start_epoch = 0 +esam_start_epoch = 8 esam_end_epoch = -1 esam_log_diagnostics = false esam_diagnostics_every = 100 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1/report.json index 9ed689981..dcf55c4c4 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/1/report.json @@ -12,12 +12,12 @@ "seed": 1, "head_selection": true, "use_esam": true, - "esam_rho": 0.005, + "esam_rho": 0.001, "esam_eps": 1e-12, - "esam_adapter_only": true, + "esam_adapter_only": false, "esam_memberwise": true, "esam_warmup_epochs": 0, - "esam_start_epoch": 0, + "esam_start_epoch": 8, "esam_end_epoch": -1, "esam_log_diagnostics": false, "esam_diagnostics_every": 100, @@ -69,16 +69,45 @@ }, "esam": { "use_esam": true, - "esam_rho": 0.005, + "esam_rho": 0.001, "esam_eps": 1e-12, - "esam_adapter_only": true, + "esam_adapter_only": false, "esam_memberwise": true, "esam_warmup_epochs": 0, - "esam_start_epoch": 0, + "esam_start_epoch": 8, "esam_end_epoch": -1, "esam_log_diagnostics": false, "esam_diagnostics_every": 100, "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, { "name": "backbone.blocks.0.0.r", "shape": [ @@ -100,6 +129,13 @@ 752 ] }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, { "name": "backbone.blocks.1.0.r", "shape": [ @@ -120,6 +156,21 @@ 32, 752 ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] } ], "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", @@ -133,42 +184,42 @@ "best_step": 1638, "metrics": { "train": { - "rmse": 0.5113957091590454, - "mae": 0.40314197540283203, - "r2": 0.5098145604133606, - "score": -0.5113957091590454 + "rmse": 0.5105330146478386, + "mae": 0.40035760402679443, + "r2": 0.5114669799804688, + "score": -0.5105330146478386 }, "val": { - "rmse": 0.5559996009180297, - "mae": 0.44189974665641785, - "r2": 0.41705596446990967, - "score": -0.5559996009180297 + "rmse": 0.5562247345576334, + "mae": 0.439607709646225, + "r2": 0.416583776473999, + "score": -0.5562247345576334 }, "test": { - "rmse": 0.5517844674752832, - "mae": 0.44495975971221924, - "r2": 0.43265408277511597, - "score": -0.5517844674752832 + "rmse": 0.550284872190118, + "mae": 0.44156232476234436, + "r2": 0.4357336759567261, + "score": -0.550284872190118 } }, - "time": "0:11:50.725269", + "time": "0:10:49.864209", "diversity": { "train": { - "mean_centered_corr": -0.020498434081673622, + "mean_centered_corr": -0.02029341645538807, "mean_pairwise_disagreement": NaN, - "member_std": 0.06282930076122284 + "member_std": 0.06168676167726517 }, "val": { - "mean_centered_corr": -0.020406439900398254, + "mean_centered_corr": -0.020232515409588814, "mean_pairwise_disagreement": NaN, - "member_std": 0.06703978031873703 + "member_std": 0.0631292313337326 }, "test": { - "mean_centered_corr": -0.020546942949295044, + "mean_centered_corr": -0.020325176417827606, "mean_pairwise_disagreement": NaN, - "member_std": 0.06463686376810074 + "member_std": 0.061680618673563004 } }, "chunk_size": null, - "eval_batch_size": 8192 + "eval_batch_size": 16384 } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml index 76f4646c4..dedbfb366 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml @@ -6,12 +6,12 @@ amp = true seed = 2 head_selection = true use_esam = true -esam_rho = 0.005 +esam_rho = 0.001 esam_eps = 1e-12 -esam_adapter_only = true +esam_adapter_only = false esam_memberwise = true esam_warmup_epochs = 0 -esam_start_epoch = 0 +esam_start_epoch = 8 esam_end_epoch = -1 esam_log_diagnostics = false esam_diagnostics_every = 100 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2/report.json index 13a846bc2..fb0de8ba8 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/2/report.json @@ -12,12 +12,12 @@ "seed": 2, "head_selection": true, "use_esam": true, - "esam_rho": 0.005, + "esam_rho": 0.001, "esam_eps": 1e-12, - "esam_adapter_only": true, + "esam_adapter_only": false, "esam_memberwise": true, "esam_warmup_epochs": 0, - "esam_start_epoch": 0, + "esam_start_epoch": 8, "esam_end_epoch": -1, "esam_log_diagnostics": false, "esam_diagnostics_every": 100, @@ -69,16 +69,45 @@ }, "esam": { "use_esam": true, - "esam_rho": 0.005, + "esam_rho": 0.001, "esam_eps": 1e-12, - "esam_adapter_only": true, + "esam_adapter_only": false, "esam_memberwise": true, "esam_warmup_epochs": 0, - "esam_start_epoch": 0, + "esam_start_epoch": 8, "esam_end_epoch": -1, "esam_log_diagnostics": false, "esam_diagnostics_every": 100, "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, { "name": "backbone.blocks.0.0.r", "shape": [ @@ -100,6 +129,13 @@ 752 ] }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, { "name": "backbone.blocks.1.0.r", "shape": [ @@ -120,6 +156,21 @@ 32, 752 ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] } ], "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", @@ -133,42 +184,42 @@ "best_step": 1638, "metrics": { "train": { - "rmse": 0.5112017866685475, - "mae": 0.402057021856308, - "r2": 0.5101861953735352, - "score": -0.5112017866685475 + "rmse": 0.5116329266744724, + "mae": 0.4038048982620239, + "r2": 0.5093597173690796, + "score": -0.5116329266744724 }, "val": { - "rmse": 0.555909194911898, - "mae": 0.44030460715293884, - "r2": 0.4172455668449402, - "score": -0.555909194911898 + "rmse": 0.5557493871736087, + "mae": 0.4417022466659546, + "r2": 0.4175804853439331, + "score": -0.5557493871736087 }, "test": { - "rmse": 0.5492662194956084, - "mae": 0.4420929551124573, - "r2": 0.43782079219818115, - "score": -0.5492662194956084 + "rmse": 0.5507763801521746, + "mae": 0.44514498114585876, + "r2": 0.43472522497177124, + "score": -0.5507763801521746 } }, - "time": "0:11:47.829319", + "time": "0:10:43.789154", "diversity": { "train": { - "mean_centered_corr": -0.020578308030962944, + "mean_centered_corr": -0.0205259807407856, "mean_pairwise_disagreement": NaN, - "member_std": 0.06090719997882843 + "member_std": 0.06119334325194359 }, "val": { - "mean_centered_corr": -0.020749453455209732, + "mean_centered_corr": -0.020698808133602142, "mean_pairwise_disagreement": NaN, - "member_std": 0.06413886696100235 + "member_std": 0.06522355228662491 }, "test": { - "mean_centered_corr": -0.020869312807917595, + "mean_centered_corr": -0.020667163655161858, "mean_pairwise_disagreement": NaN, - "member_std": 0.061883412301540375 + "member_std": 0.06273415684700012 } }, "chunk_size": null, - "eval_batch_size": 8192 + "eval_batch_size": 16384 } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0/report.json index 20d61ffa8..37c29a8bb 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/0/report.json @@ -12,12 +12,12 @@ "seed": 0, "head_selection": true, "use_esam": true, - "esam_rho": 0.005, + "esam_rho": 0.001, "esam_eps": 1e-12, - "esam_adapter_only": true, + "esam_adapter_only": false, "esam_memberwise": true, "esam_warmup_epochs": 0, - "esam_start_epoch": 0, + "esam_start_epoch": 8, "esam_end_epoch": -1, "esam_log_diagnostics": false, "esam_diagnostics_every": 100, @@ -69,16 +69,45 @@ }, "esam": { "use_esam": true, - "esam_rho": 0.005, + "esam_rho": 0.001, "esam_eps": 1e-12, - "esam_adapter_only": true, + "esam_adapter_only": false, "esam_memberwise": true, "esam_warmup_epochs": 0, - "esam_start_epoch": 0, + "esam_start_epoch": 8, "esam_end_epoch": -1, "esam_log_diagnostics": false, "esam_diagnostics_every": 100, "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, { "name": "backbone.blocks.0.0.r", "shape": [ @@ -100,6 +129,13 @@ 752 ] }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, { "name": "backbone.blocks.1.0.r", "shape": [ @@ -120,9 +156,24 @@ 32, 752 ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] } ], - "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/smoke/delivery-eta/esam_only-evaluation/_esam_diagnostics/esam_only-evaluation__0.jsonl", + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__0.jsonl", "n_diagnostics_records": 0 }, "rla_training": { @@ -130,61 +181,72 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 2184, + "best_step": 2730, "metrics": { "train": { - "rmse": 0.5047633843312425, - "mae": 0.3963008522987366, - "r2": 0.5224465727806091, - "score": -0.5047633843312425 + "rmse": 0.4981544378477528, + "mae": 0.3936895430088043, + "r2": 0.5348700881004333, + "score": -0.4981544378477528 }, "val": { - "rmse": 0.5551513290724005, - "mae": 0.4403177499771118, - "r2": 0.4188333749771118, - "score": -0.5551513290724005 + "rmse": 0.555452544543325, + "mae": 0.44208067655563354, + "r2": 0.4182025194168091, + "score": -0.555452544543325 }, "test": { - "rmse": 0.5507488105906536, - "mae": 0.44414353370666504, - "r2": 0.43478184938430786, - "score": -0.5507488105906536 + "rmse": 0.5532771704027001, + "mae": 0.44795727729797363, + "r2": 0.42958033084869385, + "score": -0.5532771704027001 } }, - "time": "0:12:51.556071", + "time": "0:13:12.089007", "diversity": { "train": { - "mean_centered_corr": -0.02027146704494953, + "mean_centered_corr": -0.020204994827508926, "mean_pairwise_disagreement": NaN, - "member_std": 0.06810352951288223 + "member_std": 0.07431785017251968 }, "val": { - "mean_centered_corr": -0.02014581859111786, + "mean_centered_corr": -0.020286401733756065, "mean_pairwise_disagreement": NaN, - "member_std": 0.07032498717308044 + "member_std": 0.07910650968551636 }, "test": { - "mean_centered_corr": -0.02043480984866619, + "mean_centered_corr": -0.02015332505106926, "mean_pairwise_disagreement": NaN, - "member_std": 0.0694202110171318 + "member_std": 0.07623301446437836 } }, "chunk_size": null, - "eval_batch_size": 8192, + "eval_batch_size": 16384, "heads": [ + 24, + 31, + 15, + 7, 13, - 21, - 10, - 6, - 29, + 8, + 1, + 27, + 26, 12, - 17, + 10, + 28, + 19, + 2, + 31, + 7, + 13, 15, + 5, 8, - 22, - 16, 21, - 4 + 1, + 4, + 31 ], - "head_selection_time": "0:00:00.394226" + "head_selection_time": "0:00:00.901734" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1/report.json index 19700c536..e9b80c6b8 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/1/report.json @@ -12,12 +12,12 @@ "seed": 1, "head_selection": true, "use_esam": true, - "esam_rho": 0.005, + "esam_rho": 0.001, "esam_eps": 1e-12, - "esam_adapter_only": true, + "esam_adapter_only": false, "esam_memberwise": true, "esam_warmup_epochs": 0, - "esam_start_epoch": 0, + "esam_start_epoch": 8, "esam_end_epoch": -1, "esam_log_diagnostics": false, "esam_diagnostics_every": 100, @@ -69,16 +69,45 @@ }, "esam": { "use_esam": true, - "esam_rho": 0.005, + "esam_rho": 0.001, "esam_eps": 1e-12, - "esam_adapter_only": true, + "esam_adapter_only": false, "esam_memberwise": true, "esam_warmup_epochs": 0, - "esam_start_epoch": 0, + "esam_start_epoch": 8, "esam_end_epoch": -1, "esam_log_diagnostics": false, "esam_diagnostics_every": 100, "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, { "name": "backbone.blocks.0.0.r", "shape": [ @@ -100,6 +129,13 @@ 752 ] }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, { "name": "backbone.blocks.1.0.r", "shape": [ @@ -120,6 +156,21 @@ 32, 752 ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] } ], "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__1.jsonl", @@ -133,56 +184,54 @@ "best_step": 1638, "metrics": { "train": { - "rmse": 0.5113803822251173, - "mae": 0.4032570719718933, - "r2": 0.5098439455032349, - "score": -0.5113803822251173 + "rmse": 0.5108096659852551, + "mae": 0.4014206528663635, + "r2": 0.5109373331069946, + "score": -0.5108096659852551 }, "val": { - "rmse": 0.5557057343261759, - "mae": 0.4411143362522125, - "r2": 0.4176720380783081, - "score": -0.5557057343261759 + "rmse": 0.5553836483690917, + "mae": 0.440256267786026, + "r2": 0.4183468818664551, + "score": -0.5553836483690917 }, "test": { - "rmse": 0.5517152752978882, - "mae": 0.4449000060558319, - "r2": 0.4327963590621948, - "score": -0.5517152752978882 + "rmse": 0.5515298017800222, + "mae": 0.4441634714603424, + "r2": 0.433177649974823, + "score": -0.5515298017800222 } }, - "time": "0:11:50.725269", + "time": "0:10:49.864209", "diversity": { "train": { - "mean_centered_corr": -0.020498434081673622, + "mean_centered_corr": -0.02029341645538807, "mean_pairwise_disagreement": NaN, - "member_std": 0.06282930076122284 + "member_std": 0.06168676167726517 }, "val": { - "mean_centered_corr": -0.020406439900398254, + "mean_centered_corr": -0.020232515409588814, "mean_pairwise_disagreement": NaN, - "member_std": 0.06703978031873703 + "member_std": 0.0631292313337326 }, "test": { - "mean_centered_corr": -0.020546942949295044, + "mean_centered_corr": -0.020325176417827606, "mean_pairwise_disagreement": NaN, - "member_std": 0.06463686376810074 + "member_std": 0.061680618673563004 } }, "chunk_size": null, - "eval_batch_size": 8192, + "eval_batch_size": 16384, "heads": [ 22, - 20, - 29, - 28, - 1, - 24, 30, - 10, + 29, 19, - 14, - 26 + 11, + 18, + 30, + 4, + 16 ], - "head_selection_time": "0:00:00.326139" + "head_selection_time": "0:00:00.270361" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2/report.json index d613ca389..29d84861d 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-greedy-heads-evaluation/2/report.json @@ -12,12 +12,12 @@ "seed": 2, "head_selection": true, "use_esam": true, - "esam_rho": 0.005, + "esam_rho": 0.001, "esam_eps": 1e-12, - "esam_adapter_only": true, + "esam_adapter_only": false, "esam_memberwise": true, "esam_warmup_epochs": 0, - "esam_start_epoch": 0, + "esam_start_epoch": 8, "esam_end_epoch": -1, "esam_log_diagnostics": false, "esam_diagnostics_every": 100, @@ -69,16 +69,45 @@ }, "esam": { "use_esam": true, - "esam_rho": 0.005, + "esam_rho": 0.001, "esam_eps": 1e-12, - "esam_adapter_only": true, + "esam_adapter_only": false, "esam_memberwise": true, "esam_warmup_epochs": 0, - "esam_start_epoch": 0, + "esam_start_epoch": 8, "esam_end_epoch": -1, "esam_log_diagnostics": false, "esam_diagnostics_every": 100, "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, { "name": "backbone.blocks.0.0.r", "shape": [ @@ -100,6 +129,13 @@ 752 ] }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, { "name": "backbone.blocks.1.0.r", "shape": [ @@ -120,6 +156,21 @@ 32, 752 ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] } ], "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_esam_only-evaluation/_esam_diagnostics/best_esam_only-evaluation__2.jsonl", @@ -133,55 +184,54 @@ "best_step": 1638, "metrics": { "train": { - "rmse": 0.5123895805314972, - "mae": 0.4043997824192047, - "r2": 0.5079073905944824, - "score": -0.5123895805314972 + "rmse": 0.5120909698461144, + "mae": 0.4043644666671753, + "r2": 0.5084807872772217, + "score": -0.5120909698461144 }, "val": { - "rmse": 0.5554111756694609, - "mae": 0.4412407875061035, - "r2": 0.4182892441749573, - "score": -0.5554111756694609 + "rmse": 0.5554437183690072, + "mae": 0.4415339529514313, + "r2": 0.41822099685668945, + "score": -0.5554437183690072 }, "test": { - "rmse": 0.5509796867295551, - "mae": 0.4447599947452545, - "r2": 0.4343078136444092, - "score": -0.5509796867295551 + "rmse": 0.5516331354574737, + "mae": 0.44613465666770935, + "r2": 0.4329652190208435, + "score": -0.5516331354574737 } }, - "time": "0:11:47.829319", + "time": "0:10:43.789154", "diversity": { "train": { - "mean_centered_corr": -0.020578308030962944, + "mean_centered_corr": -0.0205259807407856, "mean_pairwise_disagreement": NaN, - "member_std": 0.06090719997882843 + "member_std": 0.06119334325194359 }, "val": { - "mean_centered_corr": -0.020749453455209732, + "mean_centered_corr": -0.020698808133602142, "mean_pairwise_disagreement": NaN, - "member_std": 0.06413886696100235 + "member_std": 0.06522355228662491 }, "test": { - "mean_centered_corr": -0.020869312807917595, + "mean_centered_corr": -0.020667163655161858, "mean_pairwise_disagreement": NaN, - "member_std": 0.061883412301540375 + "member_std": 0.06273415684700012 } }, "chunk_size": null, - "eval_batch_size": 8192, + "eval_batch_size": 16384, "heads": [ - 28, - 11, + 8, 31, + 22, 14, - 3, + 12, 0, - 13, - 31, + 16, 27, - 16 + 13 ], - "head_selection_time": "0:00:00.290963" + "head_selection_time": "0:00:00.259911" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/0/report.json index 4d0b207d0..0ef052905 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/0/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/0/report.json @@ -37,14 +37,18 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.7, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, "mask_strength": 1.0, "anchor_fraction": 0.0, "warmup_epochs": 0, - "mask_seed": 0 + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" } }, "bins": { @@ -59,28 +63,32 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.7, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, "mask_strength": 1.0, "anchor_fraction": 0.0, "warmup_epochs": 0, - "mask_seed": 0 + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" }, "mask_stats": { - "keep_rate_actual": 0.7046874761581421, - "min_features_per_member": 142.0, - "mean_features_per_member": 155.03125, - "max_features_per_member": 168.0, - "min_members_per_feature": 16.0, - "mean_members_per_feature": 22.549999237060547, - "max_members_per_feature": 29.0, - "mean_pairwise_jaccard": 0.5446517033602734, + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, "anchor_fraction_configured": 0.0, "n_anchor_members": 0.0, - "core_fraction_configured": 0.0, - "n_core_features": 0.0 + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 } }, "cf_fisd": { @@ -156,49 +164,49 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 2730, + "best_step": 2184, "metrics": { "train": { - "rmse": 0.5070207145078325, - "mae": 0.39629772305488586, - "r2": 0.5181657075881958, - "score": -0.5070207145078325 + "rmse": 0.5081016011029494, + "mae": 0.4003114700317383, + "r2": 0.5161091685295105, + "score": -0.5081016011029494 }, "val": { - "rmse": 0.5595885535124743, - "mae": 0.4446312487125397, - "r2": 0.4095059037208557, - "score": -0.5595885535124743 + "rmse": 0.5590051856389011, + "mae": 0.4407220184803009, + "r2": 0.41073644161224365, + "score": -0.5590051856389011 }, "test": { - "rmse": 0.5534224254724212, - "mae": 0.44561949372291565, - "r2": 0.42928075790405273, - "score": -0.5534224254724212 + "rmse": 0.5534488118080276, + "mae": 0.4457874596118927, + "r2": 0.4292263984680176, + "score": -0.5534488118080276 } }, - "time": "0:09:13.835446", + "time": "0:07:43.476808", "diversity": { "train": { - "mean_centered_corr": 0.022733226418495178, + "mean_centered_corr": -0.02009296789765358, "mean_pairwise_disagreement": NaN, - "member_std": 0.12898561358451843 + "member_std": 0.07790226489305496 }, "val": { - "mean_centered_corr": 0.03887920081615448, + "mean_centered_corr": -0.02050519548356533, "mean_pairwise_disagreement": NaN, - "member_std": 0.14495275914669037 + "member_std": 0.07800902426242828 }, "test": { - "mean_centered_corr": 0.04169675335288048, + "mean_centered_corr": -0.02043369971215725, "mean_pairwise_disagreement": NaN, - "member_std": 0.1456397920846939 + "member_std": 0.07650945335626602 } }, "chunk_size": null, "eval_batch_size": 8192, "heads": [ - 8 + 6 ], - "head_selection_time": "0:00:00.023598" + "head_selection_time": "0:00:00.023785" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/1/report.json index d8bc9fda1..9d0a6ad7c 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/1/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/1/report.json @@ -37,14 +37,18 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.7, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, "mask_strength": 1.0, "anchor_fraction": 0.0, "warmup_epochs": 0, - "mask_seed": 1 + "mask_seed": 1, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" } }, "bins": { @@ -59,28 +63,32 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.7, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, "mask_strength": 1.0, "anchor_fraction": 0.0, "warmup_epochs": 0, - "mask_seed": 1 + "mask_seed": 1, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" }, "mask_stats": { - "keep_rate_actual": 0.7045454382896423, - "min_features_per_member": 142.0, - "mean_features_per_member": 155.0, - "max_features_per_member": 171.0, - "min_members_per_feature": 16.0, - "mean_members_per_feature": 22.545454025268555, - "max_members_per_feature": 29.0, - "mean_pairwise_jaccard": 0.5438212996120257, + "keep_rate_actual": 0.9758522510528564, + "min_features_per_member": 208.0, + "mean_features_per_member": 214.6875, + "max_features_per_member": 219.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.227272033691406, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9528503117194804, "anchor_fraction_configured": 0.0, "n_anchor_members": 0.0, - "core_fraction_configured": 0.0, - "n_core_features": 0.0 + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 } }, "cf_fisd": { @@ -159,46 +167,46 @@ "best_step": 1638, "metrics": { "train": { - "rmse": 0.5206705379376763, - "mae": 0.4162462055683136, - "r2": 0.49187296628952026, - "score": -0.5206705379376763 + "rmse": 0.5160363608272199, + "mae": 0.40823259949684143, + "r2": 0.5008777976036072, + "score": -0.5160363608272199 }, "val": { - "rmse": 0.5585395946793884, - "mae": 0.4492241144180298, - "r2": 0.41171765327453613, - "score": -0.5585395946793884 + "rmse": 0.5576769100347859, + "mae": 0.4429268538951874, + "r2": 0.4135335087776184, + "score": -0.5576769100347859 }, "test": { - "rmse": 0.5622948696369855, - "mae": 0.45980095863342285, - "r2": 0.4108346104621887, - "score": -0.5622948696369855 + "rmse": 0.5558933529149844, + "mae": 0.4492851793766022, + "r2": 0.42417311668395996, + "score": -0.5558933529149844 } }, - "time": "0:07:49.809972", + "time": "0:07:03.176224", "diversity": { "train": { - "mean_centered_corr": 0.0046985214576125145, + "mean_centered_corr": -0.019072668626904488, "mean_pairwise_disagreement": NaN, - "member_std": 0.09994115680456161 + "member_std": 0.07286733388900757 }, "val": { - "mean_centered_corr": 0.01109667681157589, + "mean_centered_corr": -0.019180510193109512, "mean_pairwise_disagreement": NaN, - "member_std": 0.11037178337574005 + "member_std": 0.07150229066610336 }, "test": { - "mean_centered_corr": 0.013022148050367832, + "mean_centered_corr": -0.01909942366182804, "mean_pairwise_disagreement": NaN, - "member_std": 0.1104893684387207 + "member_std": 0.07076001167297363 } }, "chunk_size": null, "eval_batch_size": 8192, "heads": [ - 18 + 22 ], - "head_selection_time": "0:00:00.023411" + "head_selection_time": "0:00:00.023446" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/2/report.json index c031c7831..c6bdbbf0a 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/2/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-best-head-evaluation/2/report.json @@ -37,14 +37,18 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.7, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, "mask_strength": 1.0, "anchor_fraction": 0.0, "warmup_epochs": 0, - "mask_seed": 2 + "mask_seed": 2, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" } }, "bins": { @@ -59,28 +63,32 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.7, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, "mask_strength": 1.0, "anchor_fraction": 0.0, "warmup_epochs": 0, - "mask_seed": 2 + "mask_seed": 2, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" }, "mask_stats": { - "keep_rate_actual": 0.7042613625526428, - "min_features_per_member": 142.0, - "mean_features_per_member": 154.9375, - "max_features_per_member": 172.0, - "min_members_per_feature": 16.0, - "mean_members_per_feature": 22.53636360168457, - "max_members_per_feature": 29.0, - "mean_pairwise_jaccard": 0.5428619561922541, + "keep_rate_actual": 0.9745738506317139, + "min_features_per_member": 207.0, + "mean_features_per_member": 214.40625, + "max_features_per_member": 219.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.186363220214844, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9505048664860526, "anchor_fraction_configured": 0.0, "n_anchor_members": 0.0, - "core_fraction_configured": 0.0, - "n_core_features": 0.0 + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 } }, "cf_fisd": { @@ -156,49 +164,49 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 3003, + "best_step": 1638, "metrics": { "train": { - "rmse": 0.5059006721989784, - "mae": 0.3979300558567047, - "r2": 0.5202922224998474, - "score": -0.5059006721989784 + "rmse": 0.5149676728055901, + "mae": 0.40653571486473083, + "r2": 0.5029430389404297, + "score": -0.5149676728055901 }, "val": { - "rmse": 0.5610582099606907, - "mae": 0.44673866033554077, - "r2": 0.406400203704834, - "score": -0.5610582099606907 + "rmse": 0.558478390179117, + "mae": 0.44451311230659485, + "r2": 0.4118465781211853, + "score": -0.558478390179117 }, "test": { - "rmse": 0.5524986222625401, - "mae": 0.4445948600769043, - "r2": 0.4311845302581787, - "score": -0.5524986222625401 + "rmse": 0.5526173066624255, + "mae": 0.4467785954475403, + "r2": 0.4309401512145996, + "score": -0.5526173066624255 } }, - "time": "0:09:37.350511", + "time": "0:07:03.506041", "diversity": { "train": { - "mean_centered_corr": 0.0061104861088097095, + "mean_centered_corr": 0.024787528440356255, "mean_pairwise_disagreement": NaN, - "member_std": 0.11999407410621643 + "member_std": 0.11345301568508148 }, "val": { - "mean_centered_corr": 0.013669072650372982, + "mean_centered_corr": 0.015488767065107822, "mean_pairwise_disagreement": NaN, - "member_std": 0.13615167140960693 + "member_std": 0.11428248137235641 }, "test": { - "mean_centered_corr": 0.0167130958288908, + "mean_centered_corr": 0.020906968042254448, "mean_pairwise_disagreement": NaN, - "member_std": 0.1356925070285797 + "member_std": 0.11407273262739182 } }, "chunk_size": null, "eval_batch_size": 8192, "heads": [ - 0 + 10 ], - "head_selection_time": "0:00:00.023116" + "head_selection_time": "0:00:00.023247" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml index 040e3d71f..ba5fa4110 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml @@ -33,7 +33,7 @@ d_embedding = 32 enabled = true mask_mode = "member_fixed" mask_granularity = "feature_group" -keep_rate = 0.7 +keep_rate = 0.975 training_only = false inverted_scaling = true use_soft_mask = false @@ -41,6 +41,10 @@ mask_strength = 1.0 anchor_fraction = 0.0 warmup_epochs = 0 mask_seed = 0 +start_epoch = 16 +start_fraction = 0.5 +group_mode = "numerical_only" +categorical_handling = "no_cat_drop" [bins] n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0/report.json index 87e75b5e0..c3225bc3f 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0/report.json @@ -37,14 +37,18 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.7, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, "mask_strength": 1.0, "anchor_fraction": 0.0, "warmup_epochs": 0, - "mask_seed": 0 + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" } }, "bins": { @@ -59,28 +63,32 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.7, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, "mask_strength": 1.0, "anchor_fraction": 0.0, "warmup_epochs": 0, - "mask_seed": 0 + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" }, "mask_stats": { - "keep_rate_actual": 0.7046874761581421, - "min_features_per_member": 142.0, - "mean_features_per_member": 155.03125, - "max_features_per_member": 168.0, - "min_members_per_feature": 16.0, - "mean_members_per_feature": 22.549999237060547, - "max_members_per_feature": 29.0, - "mean_pairwise_jaccard": 0.5446517033602734, + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, "anchor_fraction_configured": 0.0, "n_anchor_members": 0.0, - "core_fraction_configured": 0.0, - "n_core_features": 0.0 + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 } }, "cf_fisd": { @@ -156,43 +164,43 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 2730, + "best_step": 2184, "metrics": { "train": { - "rmse": 0.5028727675143494, - "mae": 0.39573734998703003, - "r2": 0.5260173082351685, - "score": -0.5028727675143494 + "rmse": 0.5047748974218713, + "mae": 0.39609476923942566, + "r2": 0.5224248170852661, + "score": -0.5047748974218713 }, "val": { - "rmse": 0.5576335149669441, - "mae": 0.4440232217311859, - "r2": 0.41362476348876953, - "score": -0.5576335149669441 + "rmse": 0.5558048331442785, + "mae": 0.43993815779685974, + "r2": 0.41746431589126587, + "score": -0.5558048331442785 }, "test": { - "rmse": 0.5507854163911843, - "mae": 0.44484779238700867, - "r2": 0.4347066879272461, - "score": -0.5507854163911843 + "rmse": 0.5501619198178428, + "mae": 0.44212156534194946, + "r2": 0.435985803604126, + "score": -0.5501619198178428 } }, - "time": "0:09:13.835446", + "time": "0:07:43.476808", "diversity": { "train": { - "mean_centered_corr": 0.022733226418495178, + "mean_centered_corr": -0.02009296789765358, "mean_pairwise_disagreement": NaN, - "member_std": 0.12898561358451843 + "member_std": 0.07790226489305496 }, "val": { - "mean_centered_corr": 0.03887920081615448, + "mean_centered_corr": -0.02050519548356533, "mean_pairwise_disagreement": NaN, - "member_std": 0.14495275914669037 + "member_std": 0.07800902426242828 }, "test": { - "mean_centered_corr": 0.04169675335288048, + "mean_centered_corr": -0.02043369971215725, "mean_pairwise_disagreement": NaN, - "member_std": 0.1456397920846939 + "member_std": 0.07650945335626602 } }, "chunk_size": null, diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml index 5dfdfdfed..27698723e 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml @@ -33,7 +33,7 @@ d_embedding = 32 enabled = true mask_mode = "member_fixed" mask_granularity = "feature_group" -keep_rate = 0.7 +keep_rate = 0.975 training_only = false inverted_scaling = true use_soft_mask = false @@ -41,6 +41,10 @@ mask_strength = 1.0 anchor_fraction = 0.0 warmup_epochs = 0 mask_seed = 1 +start_epoch = 16 +start_fraction = 0.5 +group_mode = "numerical_only" +categorical_handling = "no_cat_drop" [bins] n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1/report.json index f4a08e923..a4b87cbab 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1/report.json @@ -37,14 +37,18 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.7, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, "mask_strength": 1.0, "anchor_fraction": 0.0, "warmup_epochs": 0, - "mask_seed": 1 + "mask_seed": 1, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" } }, "bins": { @@ -59,28 +63,32 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.7, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, "mask_strength": 1.0, "anchor_fraction": 0.0, "warmup_epochs": 0, - "mask_seed": 1 + "mask_seed": 1, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" }, "mask_stats": { - "keep_rate_actual": 0.7045454382896423, - "min_features_per_member": 142.0, - "mean_features_per_member": 155.0, - "max_features_per_member": 171.0, - "min_members_per_feature": 16.0, - "mean_members_per_feature": 22.545454025268555, - "max_members_per_feature": 29.0, - "mean_pairwise_jaccard": 0.5438212996120257, + "keep_rate_actual": 0.9758522510528564, + "min_features_per_member": 208.0, + "mean_features_per_member": 214.6875, + "max_features_per_member": 219.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.227272033691406, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9528503117194804, "anchor_fraction_configured": 0.0, "n_anchor_members": 0.0, - "core_fraction_configured": 0.0, - "n_core_features": 0.0 + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 } }, "cf_fisd": { @@ -159,40 +167,40 @@ "best_step": 1638, "metrics": { "train": { - "rmse": 0.5134184759867957, - "mae": 0.40532490611076355, - "r2": 0.5059291124343872, - "score": -0.5134184759867957 + "rmse": 0.5110728431654283, + "mae": 0.4014737606048584, + "r2": 0.5104333162307739, + "score": -0.5110728431654283 }, "val": { - "rmse": 0.5557639999222178, - "mae": 0.4431096315383911, - "r2": 0.41754990816116333, - "score": -0.5557639999222178 + "rmse": 0.5559520276955289, + "mae": 0.43929943442344666, + "r2": 0.41715574264526367, + "score": -0.5559520276955289 }, "test": { - "rmse": 0.5537457867240355, - "mae": 0.44885125756263733, - "r2": 0.4286136031150818, - "score": -0.5537457867240355 + "rmse": 0.5503973468399834, + "mae": 0.44218185544013977, + "r2": 0.4355030059814453, + "score": -0.5503973468399834 } }, - "time": "0:07:49.809972", + "time": "0:07:03.176224", "diversity": { "train": { - "mean_centered_corr": 0.0046985214576125145, + "mean_centered_corr": -0.019072668626904488, "mean_pairwise_disagreement": NaN, - "member_std": 0.09994115680456161 + "member_std": 0.07286733388900757 }, "val": { - "mean_centered_corr": 0.01109667681157589, + "mean_centered_corr": -0.019180510193109512, "mean_pairwise_disagreement": NaN, - "member_std": 0.11037178337574005 + "member_std": 0.07150229066610336 }, "test": { - "mean_centered_corr": 0.013022148050367832, + "mean_centered_corr": -0.01909942366182804, "mean_pairwise_disagreement": NaN, - "member_std": 0.1104893684387207 + "member_std": 0.07076001167297363 } }, "chunk_size": null, diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml index 87637db2a..f4f39c2bd 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml @@ -33,7 +33,7 @@ d_embedding = 32 enabled = true mask_mode = "member_fixed" mask_granularity = "feature_group" -keep_rate = 0.7 +keep_rate = 0.975 training_only = false inverted_scaling = true use_soft_mask = false @@ -41,6 +41,10 @@ mask_strength = 1.0 anchor_fraction = 0.0 warmup_epochs = 0 mask_seed = 2 +start_epoch = 16 +start_fraction = 0.5 +group_mode = "numerical_only" +categorical_handling = "no_cat_drop" [bins] n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2/report.json index ab77cd273..fff0b3972 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2/report.json @@ -37,14 +37,18 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.7, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, "mask_strength": 1.0, "anchor_fraction": 0.0, "warmup_epochs": 0, - "mask_seed": 2 + "mask_seed": 2, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" } }, "bins": { @@ -59,28 +63,32 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.7, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, "mask_strength": 1.0, "anchor_fraction": 0.0, "warmup_epochs": 0, - "mask_seed": 2 + "mask_seed": 2, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" }, "mask_stats": { - "keep_rate_actual": 0.7042613625526428, - "min_features_per_member": 142.0, - "mean_features_per_member": 154.9375, - "max_features_per_member": 172.0, - "min_members_per_feature": 16.0, - "mean_members_per_feature": 22.53636360168457, - "max_members_per_feature": 29.0, - "mean_pairwise_jaccard": 0.5428619561922541, + "keep_rate_actual": 0.9745738506317139, + "min_features_per_member": 207.0, + "mean_features_per_member": 214.40625, + "max_features_per_member": 219.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.186363220214844, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9505048664860526, "anchor_fraction_configured": 0.0, "n_anchor_members": 0.0, - "core_fraction_configured": 0.0, - "n_core_features": 0.0 + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 } }, "cf_fisd": { @@ -156,43 +164,43 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 3003, + "best_step": 1638, "metrics": { "train": { - "rmse": 0.49698357865243403, - "mae": 0.39058470726013184, - "r2": 0.5370539426803589, - "score": -0.49698357865243403 + "rmse": 0.5118594085498649, + "mae": 0.4030582904815674, + "r2": 0.508925199508667, + "score": -0.5118594085498649 }, "val": { - "rmse": 0.556207481645197, - "mae": 0.4430869519710541, - "r2": 0.4166199564933777, - "score": -0.556207481645197 + "rmse": 0.5562502378827562, + "mae": 0.4414753019809723, + "r2": 0.4165303111076355, + "score": -0.5562502378827562 }, "test": { - "rmse": 0.551272504610975, - "mae": 0.4455644190311432, - "r2": 0.4337064027786255, - "score": -0.551272504610975 + "rmse": 0.5500394546922427, + "mae": 0.44385185837745667, + "r2": 0.4362368583679199, + "score": -0.5500394546922427 } }, - "time": "0:09:37.350511", + "time": "0:07:03.506041", "diversity": { "train": { - "mean_centered_corr": 0.0061104861088097095, + "mean_centered_corr": 0.024787528440356255, "mean_pairwise_disagreement": NaN, - "member_std": 0.11999407410621643 + "member_std": 0.11345301568508148 }, "val": { - "mean_centered_corr": 0.013669072650372982, + "mean_centered_corr": 0.015488767065107822, "mean_pairwise_disagreement": NaN, - "member_std": 0.13615167140960693 + "member_std": 0.11428248137235641 }, "test": { - "mean_centered_corr": 0.0167130958288908, + "mean_centered_corr": 0.020906968042254448, "mean_pairwise_disagreement": NaN, - "member_std": 0.1356925070285797 + "member_std": 0.11407273262739182 } }, "chunk_size": null, diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0/report.json index 22735cb5d..798a74fc5 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/0/report.json @@ -37,14 +37,18 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.7, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, "mask_strength": 1.0, "anchor_fraction": 0.0, "warmup_epochs": 0, - "mask_seed": 0 + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" } }, "bins": { @@ -59,28 +63,32 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.7, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, "mask_strength": 1.0, "anchor_fraction": 0.0, "warmup_epochs": 0, - "mask_seed": 0 + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" }, "mask_stats": { - "keep_rate_actual": 0.7046874761581421, - "min_features_per_member": 142.0, - "mean_features_per_member": 155.03125, - "max_features_per_member": 168.0, - "min_members_per_feature": 16.0, - "mean_members_per_feature": 22.549999237060547, - "max_members_per_feature": 29.0, - "mean_pairwise_jaccard": 0.5446517033602734, + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, "anchor_fraction_configured": 0.0, "n_anchor_members": 0.0, - "core_fraction_configured": 0.0, - "n_core_features": 0.0 + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 } }, "cf_fisd": { @@ -156,56 +164,61 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 2730, + "best_step": 2184, "metrics": { "train": { - "rmse": 0.503753402357704, - "mae": 0.3957558870315552, - "r2": 0.5243557691574097, - "score": -0.503753402357704 + "rmse": 0.5051252716316714, + "mae": 0.39751043915748596, + "r2": 0.5217615962028503, + "score": -0.5051252716316714 }, "val": { - "rmse": 0.5551008108444078, - "mae": 0.43975356221199036, - "r2": 0.41893917322158813, - "score": -0.5551008108444078 + "rmse": 0.5553099940581016, + "mae": 0.44019240140914917, + "r2": 0.4185011386871338, + "score": -0.5553099940581016 }, "test": { - "rmse": 0.5501919833980724, - "mae": 0.44342172145843506, - "r2": 0.4359241724014282, - "score": -0.5501919833980724 + "rmse": 0.5510631408064661, + "mae": 0.44413861632347107, + "r2": 0.43413645029067993, + "score": -0.5510631408064661 } }, - "time": "0:09:13.835446", + "time": "0:07:43.476808", "diversity": { "train": { - "mean_centered_corr": 0.022733226418495178, + "mean_centered_corr": -0.02009296789765358, "mean_pairwise_disagreement": NaN, - "member_std": 0.12898561358451843 + "member_std": 0.07790226489305496 }, "val": { - "mean_centered_corr": 0.03887920081615448, + "mean_centered_corr": -0.02050519548356533, "mean_pairwise_disagreement": NaN, - "member_std": 0.14495275914669037 + "member_std": 0.07800902426242828 }, "test": { - "mean_centered_corr": 0.04169675335288048, + "mean_centered_corr": -0.02043369971215725, "mean_pairwise_disagreement": NaN, - "member_std": 0.1456397920846939 + "member_std": 0.07650945335626602 } }, "chunk_size": null, "eval_batch_size": 8192, "heads": [ + 6, + 13, + 26, + 10, + 14, 8, - 11, - 4, - 16, - 2, 7, - 23, - 18 + 27, + 17, + 26, + 13, + 25, + 19 ], - "head_selection_time": "0:00:00.227173" + "head_selection_time": "0:00:00.388499" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1/report.json index e6e14c6ab..c2a3902a9 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/1/report.json @@ -37,14 +37,18 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.7, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, "mask_strength": 1.0, "anchor_fraction": 0.0, "warmup_epochs": 0, - "mask_seed": 1 + "mask_seed": 1, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" } }, "bins": { @@ -59,28 +63,32 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.7, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, "mask_strength": 1.0, "anchor_fraction": 0.0, "warmup_epochs": 0, - "mask_seed": 1 + "mask_seed": 1, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" }, "mask_stats": { - "keep_rate_actual": 0.7045454382896423, - "min_features_per_member": 142.0, - "mean_features_per_member": 155.0, - "max_features_per_member": 171.0, - "min_members_per_feature": 16.0, - "mean_members_per_feature": 22.545454025268555, - "max_members_per_feature": 29.0, - "mean_pairwise_jaccard": 0.5438212996120257, + "keep_rate_actual": 0.9758522510528564, + "min_features_per_member": 208.0, + "mean_features_per_member": 214.6875, + "max_features_per_member": 219.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.227272033691406, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9528503117194804, "anchor_fraction_configured": 0.0, "n_anchor_members": 0.0, - "core_fraction_configured": 0.0, - "n_core_features": 0.0 + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 } }, "cf_fisd": { @@ -159,53 +167,54 @@ "best_step": 1638, "metrics": { "train": { - "rmse": 0.5131697140167799, - "mae": 0.40394970774650574, - "r2": 0.5064077377319336, - "score": -0.5131697140167799 + "rmse": 0.5118196984810967, + "mae": 0.404056578874588, + "r2": 0.5090013742446899, + "score": -0.5118196984810967 }, "val": { - "rmse": 0.5549583042651809, - "mae": 0.4405714273452759, - "r2": 0.41923749446868896, - "score": -0.5549583042651809 + "rmse": 0.55527543085353, + "mae": 0.4401775598526001, + "r2": 0.4185735583305359, + "score": -0.55527543085353 }, "test": { - "rmse": 0.5533328640985606, - "mae": 0.4472924470901489, - "r2": 0.42946547269821167, - "score": -0.5533328640985606 + "rmse": 0.5524079939671243, + "mae": 0.44586870074272156, + "r2": 0.43137115240097046, + "score": -0.5524079939671243 } }, - "time": "0:07:49.809972", + "time": "0:07:03.176224", "diversity": { "train": { - "mean_centered_corr": 0.0046985214576125145, + "mean_centered_corr": -0.019072668626904488, "mean_pairwise_disagreement": NaN, - "member_std": 0.09994115680456161 + "member_std": 0.07286733388900757 }, "val": { - "mean_centered_corr": 0.01109667681157589, + "mean_centered_corr": -0.019180510193109512, "mean_pairwise_disagreement": NaN, - "member_std": 0.11037178337574005 + "member_std": 0.07150229066610336 }, "test": { - "mean_centered_corr": 0.013022148050367832, + "mean_centered_corr": -0.01909942366182804, "mean_pairwise_disagreement": NaN, - "member_std": 0.1104893684387207 + "member_std": 0.07076001167297363 } }, "chunk_size": null, "eval_batch_size": 8192, "heads": [ - 18, - 11, 22, - 24, + 30, 4, - 12, + 21, + 18, + 19, + 23, 30, - 9 + 1 ], - "head_selection_time": "0:00:00.233733" + "head_selection_time": "0:00:00.258532" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2/report.json index 08f856a37..5eb17c255 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_mfb_only-greedy-heads-evaluation/2/report.json @@ -37,14 +37,18 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.7, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, "mask_strength": 1.0, "anchor_fraction": 0.0, "warmup_epochs": 0, - "mask_seed": 2 + "mask_seed": 2, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" } }, "bins": { @@ -59,28 +63,32 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.7, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, "mask_strength": 1.0, "anchor_fraction": 0.0, "warmup_epochs": 0, - "mask_seed": 2 + "mask_seed": 2, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" }, "mask_stats": { - "keep_rate_actual": 0.7042613625526428, - "min_features_per_member": 142.0, - "mean_features_per_member": 154.9375, - "max_features_per_member": 172.0, - "min_members_per_feature": 16.0, - "mean_members_per_feature": 22.53636360168457, - "max_members_per_feature": 29.0, - "mean_pairwise_jaccard": 0.5428619561922541, + "keep_rate_actual": 0.9745738506317139, + "min_features_per_member": 207.0, + "mean_features_per_member": 214.40625, + "max_features_per_member": 219.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.186363220214844, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9505048664860526, "anchor_fraction_configured": 0.0, "n_anchor_members": 0.0, - "core_fraction_configured": 0.0, - "n_core_features": 0.0 + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 } }, "cf_fisd": { @@ -156,65 +164,68 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 3003, + "best_step": 1638, "metrics": { "train": { - "rmse": 0.4972497582432623, - "mae": 0.3897043466567993, - "r2": 0.5365579128265381, - "score": -0.4972497582432623 + "rmse": 0.5126941208598502, + "mae": 0.404828816652298, + "r2": 0.5073222517967224, + "score": -0.5126941208598502 }, "val": { - "rmse": 0.55517701590878, - "mae": 0.4405524730682373, - "r2": 0.4187796115875244, - "score": -0.55517701590878 + "rmse": 0.5553606274153348, + "mae": 0.4412638247013092, + "r2": 0.4183950424194336, + "score": -0.5553606274153348 }, "test": { - "rmse": 0.5505736480564649, - "mae": 0.4438551664352417, - "r2": 0.43514126539230347, - "score": -0.5505736480564649 + "rmse": 0.5524157087402433, + "mae": 0.4469607174396515, + "r2": 0.43135523796081543, + "score": -0.5524157087402433 } }, - "time": "0:09:37.350511", + "time": "0:07:03.506041", "diversity": { "train": { - "mean_centered_corr": 0.0061104861088097095, + "mean_centered_corr": 0.024787528440356255, "mean_pairwise_disagreement": NaN, - "member_std": 0.11999407410621643 + "member_std": 0.11345301568508148 }, "val": { - "mean_centered_corr": 0.013669072650372982, + "mean_centered_corr": 0.015488767065107822, "mean_pairwise_disagreement": NaN, - "member_std": 0.13615167140960693 + "member_std": 0.11428248137235641 }, "test": { - "mean_centered_corr": 0.0167130958288908, + "mean_centered_corr": 0.020906968042254448, "mean_pairwise_disagreement": NaN, - "member_std": 0.1356925070285797 + "member_std": 0.11407273262739182 } }, "chunk_size": null, "eval_batch_size": 8192, "heads": [ - 0, - 23, - 13, - 7, - 5, - 11, - 8, - 30, + 10, 14, - 26, - 3, 4, + 6, + 27, 13, + 8, + 31, + 16, + 12, + 14, + 25, 28, - 5, - 7, - 23 + 4, + 1, + 27, + 14, + 31, + 4, + 6 ], - "head_selection_time": "0:00:00.548122" + "head_selection_time": "0:00:00.694336" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0/report.json index c496cfa72..f6fc58a42 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/0/report.json @@ -11,7 +11,7 @@ "amp": true, "seed": 0, "head_selection": true, - "rla_adapter_lr_multiplier": 1.0, + "rla_adapter_lr_multiplier": 0.5, "rla_extra_paths_freeze_fraction": 0.0, "data": { "cache": true, @@ -26,11 +26,11 @@ "arch_type": "tabm", "k": 32, "share_training_batches": false, - "rla_rank": 2, + "rla_rank": 4, "rla_first_only": false, "rla_additive": false, "rla_init": "base_preserving", - "rla_base_preserve_noise": 0.001, + "rla_base_preserve_noise": 0.0001, "backbone": { "n_blocks": 2, "d_block": 752, @@ -45,7 +45,7 @@ "n_bins": 81 } }, - "n_parameters": 7056800, + "n_parameters": 7647904, "prediction_type": "labels", "mfb": { "enabled": false, @@ -81,7 +81,7 @@ "shape": [ 32, 6980, - 2 + 4 ] }, { @@ -89,7 +89,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -104,7 +104,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -112,7 +112,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -127,53 +127,53 @@ "n_diagnostics_records": 0 }, "rla_training": { - "adapter_lr_multiplier": 1.0, + "adapter_lr_multiplier": 0.5, "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, "best_step": 1365, "metrics": { "train": { - "rmse": 0.5169435461643365, - "mae": 0.4078221619129181, - "r2": 0.49912136793136597, - "score": -0.5169435461643365 + "rmse": 0.5175918577279733, + "mae": 0.40775081515312195, + "r2": 0.4978642463684082, + "score": -0.5175918577279733 }, "val": { - "rmse": 0.5574634025946379, - "mae": 0.44402608275413513, - "r2": 0.41398245096206665, - "score": -0.5574634025946379 + "rmse": 0.5572652691682626, + "mae": 0.44076380133628845, + "r2": 0.4143989086151123, + "score": -0.5572652691682626 }, "test": { - "rmse": 0.5547066537478277, - "mae": 0.450179785490036, - "r2": 0.4266290068626404, - "score": -0.5547066537478277 + "rmse": 0.5524207799310179, + "mae": 0.4452172815799713, + "r2": 0.4313448667526245, + "score": -0.5524207799310179 } }, - "time": "0:09:33.111358", + "time": "0:14:43.732239", "diversity": { "train": { - "mean_centered_corr": -0.020444847643375397, + "mean_centered_corr": -0.02016017585992813, "mean_pairwise_disagreement": NaN, - "member_std": 0.059093035757541656 + "member_std": 0.06105273962020874 }, "val": { - "mean_centered_corr": -0.020575184375047684, + "mean_centered_corr": -0.01993357017636299, "mean_pairwise_disagreement": NaN, - "member_std": 0.06087604537606239 + "member_std": 0.06221357360482216 }, "test": { - "mean_centered_corr": -0.020676201209425926, + "mean_centered_corr": -0.020221160724759102, "mean_pairwise_disagreement": NaN, - "member_std": 0.060276810079813004 + "member_std": 0.06099524348974228 } }, "chunk_size": null, "eval_batch_size": 8192, "heads": [ - 18 + 11 ], - "head_selection_time": "0:00:00.023373" + "head_selection_time": "0:00:00.024300" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1/report.json index de4494e93..b01d46298 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/1/report.json @@ -11,7 +11,7 @@ "amp": true, "seed": 1, "head_selection": true, - "rla_adapter_lr_multiplier": 1.0, + "rla_adapter_lr_multiplier": 0.5, "rla_extra_paths_freeze_fraction": 0.0, "data": { "cache": true, @@ -26,11 +26,11 @@ "arch_type": "tabm", "k": 32, "share_training_batches": false, - "rla_rank": 2, + "rla_rank": 4, "rla_first_only": false, "rla_additive": false, "rla_init": "base_preserving", - "rla_base_preserve_noise": 0.001, + "rla_base_preserve_noise": 0.0001, "backbone": { "n_blocks": 2, "d_block": 752, @@ -45,7 +45,7 @@ "n_bins": 81 } }, - "n_parameters": 7056800, + "n_parameters": 7647904, "prediction_type": "labels", "mfb": { "enabled": false, @@ -81,7 +81,7 @@ "shape": [ 32, 6980, - 2 + 4 ] }, { @@ -89,7 +89,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -104,7 +104,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -112,7 +112,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -127,47 +127,47 @@ "n_diagnostics_records": 0 }, "rla_training": { - "adapter_lr_multiplier": 1.0, + "adapter_lr_multiplier": 0.5, "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, "best_step": 1365, "metrics": { "train": { - "rmse": 0.516247633496788, - "mae": 0.40711885690689087, - "r2": 0.5004690289497375, - "score": -0.516247633496788 + "rmse": 0.5171067304438404, + "mae": 0.40677744150161743, + "r2": 0.49880504608154297, + "score": -0.5171067304438404 }, "val": { - "rmse": 0.5576988200546102, - "mae": 0.4442465603351593, - "r2": 0.41348737478256226, - "score": -0.5576988200546102 + "rmse": 0.5577909128836495, + "mae": 0.44260263442993164, + "r2": 0.41329365968704224, + "score": -0.5577909128836495 }, "test": { - "rmse": 0.551726996987587, - "mae": 0.44564470648765564, - "r2": 0.43277227878570557, - "score": -0.551726996987587 + "rmse": 0.5532065757037241, + "mae": 0.44593191146850586, + "r2": 0.4297259449958801, + "score": -0.5532065757037241 } }, - "time": "0:09:34.536942", + "time": "0:14:46.412006", "diversity": { "train": { - "mean_centered_corr": -0.020366203039884567, + "mean_centered_corr": -0.020146815106272697, "mean_pairwise_disagreement": NaN, - "member_std": 0.05432252958416939 + "member_std": 0.061014287173748016 }, "val": { - "mean_centered_corr": -0.020544419065117836, + "mean_centered_corr": -0.020077988505363464, "mean_pairwise_disagreement": NaN, - "member_std": 0.054353535175323486 + "member_std": 0.06329856067895889 }, "test": { - "mean_centered_corr": -0.02033921517431736, + "mean_centered_corr": -0.01972520351409912, "mean_pairwise_disagreement": NaN, - "member_std": 0.05365084111690521 + "member_std": 0.06241019815206528 } }, "chunk_size": null, @@ -175,5 +175,5 @@ "heads": [ 5 ], - "head_selection_time": "0:00:00.023071" + "head_selection_time": "0:00:00.023596" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2/report.json index 3123c6e12..7c2f9848a 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-best-head-evaluation/2/report.json @@ -11,7 +11,7 @@ "amp": true, "seed": 2, "head_selection": true, - "rla_adapter_lr_multiplier": 1.0, + "rla_adapter_lr_multiplier": 0.5, "rla_extra_paths_freeze_fraction": 0.0, "data": { "cache": true, @@ -26,11 +26,11 @@ "arch_type": "tabm", "k": 32, "share_training_batches": false, - "rla_rank": 2, + "rla_rank": 4, "rla_first_only": false, "rla_additive": false, "rla_init": "base_preserving", - "rla_base_preserve_noise": 0.001, + "rla_base_preserve_noise": 0.0001, "backbone": { "n_blocks": 2, "d_block": 752, @@ -45,7 +45,7 @@ "n_bins": 81 } }, - "n_parameters": 7056800, + "n_parameters": 7647904, "prediction_type": "labels", "mfb": { "enabled": false, @@ -81,7 +81,7 @@ "shape": [ 32, 6980, - 2 + 4 ] }, { @@ -89,7 +89,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -104,7 +104,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -112,7 +112,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -127,53 +127,53 @@ "n_diagnostics_records": 0 }, "rla_training": { - "adapter_lr_multiplier": 1.0, + "adapter_lr_multiplier": 0.5, "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 1365, + "best_step": 1911, "metrics": { "train": { - "rmse": 0.5148993790576664, - "mae": 0.40408939123153687, - "r2": 0.5030747652053833, - "score": -0.5148993790576664 + "rmse": 0.5137978750972679, + "mae": 0.4069885015487671, + "r2": 0.50519859790802, + "score": -0.5137978750972679 }, "val": { - "rmse": 0.5580165521873937, - "mae": 0.4407314658164978, - "r2": 0.41281890869140625, - "score": -0.5580165521873937 + "rmse": 0.5585470113340857, + "mae": 0.44582831859588623, + "r2": 0.4117019772529602, + "score": -0.5585470113340857 }, "test": { - "rmse": 0.5499792279309027, - "mae": 0.44245660305023193, - "r2": 0.43636035919189453, - "score": -0.5499792279309027 + "rmse": 0.5573737419749005, + "mae": 0.4522700607776642, + "r2": 0.4211021065711975, + "score": -0.5573737419749005 } }, - "time": "0:09:31.116943", + "time": "0:16:06.202820", "diversity": { "train": { - "mean_centered_corr": -0.020301729440689087, + "mean_centered_corr": -0.020242199301719666, "mean_pairwise_disagreement": NaN, - "member_std": 0.05993633717298508 + "member_std": 0.0667627602815628 }, "val": { - "mean_centered_corr": -0.020580198615789413, + "mean_centered_corr": -0.02033313177525997, "mean_pairwise_disagreement": NaN, - "member_std": 0.06198854371905327 + "member_std": 0.06825815886259079 }, "test": { - "mean_centered_corr": -0.02039889432489872, + "mean_centered_corr": -0.020218119025230408, "mean_pairwise_disagreement": NaN, - "member_std": 0.05979244410991669 + "member_std": 0.06682927906513214 } }, "chunk_size": null, "eval_batch_size": 8192, "heads": [ - 13 + 5 ], - "head_selection_time": "0:00:00.023734" + "head_selection_time": "0:00:00.024826" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml index 6d6f2bc2b..f15a53b07 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml @@ -5,7 +5,7 @@ gradient_clipping_norm = 1.0 amp = true seed = 0 head_selection = true -rla_adapter_lr_multiplier = 1.0 +rla_adapter_lr_multiplier = 0.5 rla_extra_paths_freeze_fraction = 0.0 [data] @@ -21,11 +21,11 @@ weight_decay = 0.0507074044872309 arch_type = "tabm" k = 32 share_training_batches = false -rla_rank = 2 +rla_rank = 4 rla_first_only = false rla_additive = false rla_init = "base_preserving" -rla_base_preserve_noise = 0.001 +rla_base_preserve_noise = 0.0001 [model.backbone] n_blocks = 2 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0/report.json index 03a93045c..9c3fd8d6d 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/0/report.json @@ -11,7 +11,7 @@ "amp": true, "seed": 0, "head_selection": true, - "rla_adapter_lr_multiplier": 1.0, + "rla_adapter_lr_multiplier": 0.5, "rla_extra_paths_freeze_fraction": 0.0, "data": { "cache": true, @@ -26,11 +26,11 @@ "arch_type": "tabm", "k": 32, "share_training_batches": false, - "rla_rank": 2, + "rla_rank": 4, "rla_first_only": false, "rla_additive": false, "rla_init": "base_preserving", - "rla_base_preserve_noise": 0.001, + "rla_base_preserve_noise": 0.0001, "backbone": { "n_blocks": 2, "d_block": 752, @@ -45,7 +45,7 @@ "n_bins": 81 } }, - "n_parameters": 7056800, + "n_parameters": 7647904, "prediction_type": "labels", "mfb": { "enabled": false, @@ -81,7 +81,7 @@ "shape": [ 32, 6980, - 2 + 4 ] }, { @@ -89,7 +89,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -104,7 +104,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -112,7 +112,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -127,47 +127,47 @@ "n_diagnostics_records": 0 }, "rla_training": { - "adapter_lr_multiplier": 1.0, + "adapter_lr_multiplier": 0.5, "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, "best_step": 1365, "metrics": { "train": { - "rmse": 0.5137012893899691, - "mae": 0.40439915657043457, - "r2": 0.5053846836090088, - "score": -0.5137012893899691 + "rmse": 0.5150130136938161, + "mae": 0.40628108382225037, + "r2": 0.5028554797172546, + "score": -0.5150130136938161 }, "val": { - "rmse": 0.5556336245387727, - "mae": 0.4420306384563446, - "r2": 0.4178231954574585, - "score": -0.5556336245387727 + "rmse": 0.5559356776481553, + "mae": 0.441616415977478, + "r2": 0.4171900153160095, + "score": -0.5559356776481553 }, "test": { - "rmse": 0.5518591323593222, - "mae": 0.4463585913181305, - "r2": 0.43250054121017456, - "score": -0.5518591323593222 + "rmse": 0.55135486033622, + "mae": 0.445721298456192, + "r2": 0.43353718519210815, + "score": -0.55135486033622 } }, - "time": "0:09:33.111358", + "time": "0:14:43.732239", "diversity": { "train": { - "mean_centered_corr": -0.020444847643375397, + "mean_centered_corr": -0.02016017585992813, "mean_pairwise_disagreement": NaN, - "member_std": 0.059093035757541656 + "member_std": 0.06105273962020874 }, "val": { - "mean_centered_corr": -0.020575184375047684, + "mean_centered_corr": -0.01993357017636299, "mean_pairwise_disagreement": NaN, - "member_std": 0.06087604537606239 + "member_std": 0.06221357360482216 }, "test": { - "mean_centered_corr": -0.020676201209425926, + "mean_centered_corr": -0.020221160724759102, "mean_pairwise_disagreement": NaN, - "member_std": 0.060276810079813004 + "member_std": 0.06099524348974228 } }, "chunk_size": null, diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml index d02cc3fe9..19367e222 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml @@ -5,7 +5,7 @@ gradient_clipping_norm = 1.0 amp = true seed = 1 head_selection = true -rla_adapter_lr_multiplier = 1.0 +rla_adapter_lr_multiplier = 0.5 rla_extra_paths_freeze_fraction = 0.0 [data] @@ -21,11 +21,11 @@ weight_decay = 0.0507074044872309 arch_type = "tabm" k = 32 share_training_batches = false -rla_rank = 2 +rla_rank = 4 rla_first_only = false rla_additive = false rla_init = "base_preserving" -rla_base_preserve_noise = 0.001 +rla_base_preserve_noise = 0.0001 [model.backbone] n_blocks = 2 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1/report.json index b0272ba3f..941521617 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/1/report.json @@ -11,7 +11,7 @@ "amp": true, "seed": 1, "head_selection": true, - "rla_adapter_lr_multiplier": 1.0, + "rla_adapter_lr_multiplier": 0.5, "rla_extra_paths_freeze_fraction": 0.0, "data": { "cache": true, @@ -26,11 +26,11 @@ "arch_type": "tabm", "k": 32, "share_training_batches": false, - "rla_rank": 2, + "rla_rank": 4, "rla_first_only": false, "rla_additive": false, "rla_init": "base_preserving", - "rla_base_preserve_noise": 0.001, + "rla_base_preserve_noise": 0.0001, "backbone": { "n_blocks": 2, "d_block": 752, @@ -45,7 +45,7 @@ "n_bins": 81 } }, - "n_parameters": 7056800, + "n_parameters": 7647904, "prediction_type": "labels", "mfb": { "enabled": false, @@ -81,7 +81,7 @@ "shape": [ 32, 6980, - 2 + 4 ] }, { @@ -89,7 +89,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -104,7 +104,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -112,7 +112,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -127,47 +127,47 @@ "n_diagnostics_records": 0 }, "rla_training": { - "adapter_lr_multiplier": 1.0, + "adapter_lr_multiplier": 0.5, "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, "best_step": 1365, "metrics": { "train": { - "rmse": 0.5138557888087629, - "mae": 0.4044031500816345, - "r2": 0.50508713722229, - "score": -0.5138557888087629 + "rmse": 0.5138602835910225, + "mae": 0.4030267894268036, + "r2": 0.5050784349441528, + "score": -0.5138602835910225 }, "val": { - "rmse": 0.5562834011722252, - "mae": 0.44168752431869507, - "r2": 0.41646069288253784, - "score": -0.5562834011722252 + "rmse": 0.5564338969448529, + "mae": 0.43967634439468384, + "r2": 0.4161449074745178, + "score": -0.5564338969448529 }, "test": { - "rmse": 0.5505008118837418, - "mae": 0.44447061419487, - "r2": 0.43529075384140015, - "score": -0.5505008118837418 + "rmse": 0.5491088203688107, + "mae": 0.4416921138763428, + "r2": 0.43814295530319214, + "score": -0.5491088203688107 } }, - "time": "0:09:34.536942", + "time": "0:14:46.412006", "diversity": { "train": { - "mean_centered_corr": -0.020366203039884567, + "mean_centered_corr": -0.020146815106272697, "mean_pairwise_disagreement": NaN, - "member_std": 0.05432252958416939 + "member_std": 0.061014287173748016 }, "val": { - "mean_centered_corr": -0.020544419065117836, + "mean_centered_corr": -0.020077988505363464, "mean_pairwise_disagreement": NaN, - "member_std": 0.054353535175323486 + "member_std": 0.06329856067895889 }, "test": { - "mean_centered_corr": -0.02033921517431736, + "mean_centered_corr": -0.01972520351409912, "mean_pairwise_disagreement": NaN, - "member_std": 0.05365084111690521 + "member_std": 0.06241019815206528 } }, "chunk_size": null, diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml index 6e4fdc141..94b60b1ef 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml @@ -5,7 +5,7 @@ gradient_clipping_norm = 1.0 amp = true seed = 2 head_selection = true -rla_adapter_lr_multiplier = 1.0 +rla_adapter_lr_multiplier = 0.5 rla_extra_paths_freeze_fraction = 0.0 [data] @@ -21,11 +21,11 @@ weight_decay = 0.0507074044872309 arch_type = "tabm" k = 32 share_training_batches = false -rla_rank = 2 +rla_rank = 4 rla_first_only = false rla_additive = false rla_init = "base_preserving" -rla_base_preserve_noise = 0.001 +rla_base_preserve_noise = 0.0001 [model.backbone] n_blocks = 2 diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2/report.json index 2cde9fc4f..33f34dd36 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-evaluation/2/report.json @@ -11,7 +11,7 @@ "amp": true, "seed": 2, "head_selection": true, - "rla_adapter_lr_multiplier": 1.0, + "rla_adapter_lr_multiplier": 0.5, "rla_extra_paths_freeze_fraction": 0.0, "data": { "cache": true, @@ -26,11 +26,11 @@ "arch_type": "tabm", "k": 32, "share_training_batches": false, - "rla_rank": 2, + "rla_rank": 4, "rla_first_only": false, "rla_additive": false, "rla_init": "base_preserving", - "rla_base_preserve_noise": 0.001, + "rla_base_preserve_noise": 0.0001, "backbone": { "n_blocks": 2, "d_block": 752, @@ -45,7 +45,7 @@ "n_bins": 81 } }, - "n_parameters": 7056800, + "n_parameters": 7647904, "prediction_type": "labels", "mfb": { "enabled": false, @@ -81,7 +81,7 @@ "shape": [ 32, 6980, - 2 + 4 ] }, { @@ -89,7 +89,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -104,7 +104,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -112,7 +112,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -127,47 +127,47 @@ "n_diagnostics_records": 0 }, "rla_training": { - "adapter_lr_multiplier": 1.0, + "adapter_lr_multiplier": 0.5, "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 1365, + "best_step": 1911, "metrics": { "train": { - "rmse": 0.5124919088236671, - "mae": 0.40055063366889954, - "r2": 0.5077108144760132, - "score": -0.5124919088236671 + "rmse": 0.5080919523858995, + "mae": 0.3977190852165222, + "r2": 0.5161275267601013, + "score": -0.5080919523858995 }, "val": { - "rmse": 0.5564119371157284, - "mae": 0.4400753974914551, - "r2": 0.416191041469574, - "score": -0.5564119371157284 + "rmse": 0.5562272527951271, + "mae": 0.43906888365745544, + "r2": 0.416578471660614, + "score": -0.5562272527951271 }, "test": { - "rmse": 0.5491394300416609, - "mae": 0.44087520241737366, - "r2": 0.4380803108215332, - "score": -0.5491394300416609 + "rmse": 0.5494697322909359, + "mae": 0.44080713391304016, + "r2": 0.43740415573120117, + "score": -0.5494697322909359 } }, - "time": "0:09:31.116943", + "time": "0:16:06.202820", "diversity": { "train": { - "mean_centered_corr": -0.020301729440689087, + "mean_centered_corr": -0.020242199301719666, "mean_pairwise_disagreement": NaN, - "member_std": 0.05993633717298508 + "member_std": 0.0667627602815628 }, "val": { - "mean_centered_corr": -0.020580198615789413, + "mean_centered_corr": -0.02033313177525997, "mean_pairwise_disagreement": NaN, - "member_std": 0.06198854371905327 + "member_std": 0.06825815886259079 }, "test": { - "mean_centered_corr": -0.02039889432489872, + "mean_centered_corr": -0.020218119025230408, "mean_pairwise_disagreement": NaN, - "member_std": 0.05979244410991669 + "member_std": 0.06682927906513214 } }, "chunk_size": null, diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0/report.json index 882c6c418..043c34d52 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/0/report.json @@ -11,7 +11,7 @@ "amp": true, "seed": 0, "head_selection": true, - "rla_adapter_lr_multiplier": 1.0, + "rla_adapter_lr_multiplier": 0.5, "rla_extra_paths_freeze_fraction": 0.0, "data": { "cache": true, @@ -26,11 +26,11 @@ "arch_type": "tabm", "k": 32, "share_training_batches": false, - "rla_rank": 2, + "rla_rank": 4, "rla_first_only": false, "rla_additive": false, "rla_init": "base_preserving", - "rla_base_preserve_noise": 0.001, + "rla_base_preserve_noise": 0.0001, "backbone": { "n_blocks": 2, "d_block": 752, @@ -45,7 +45,7 @@ "n_bins": 81 } }, - "n_parameters": 7056800, + "n_parameters": 7647904, "prediction_type": "labels", "mfb": { "enabled": false, @@ -81,7 +81,7 @@ "shape": [ 32, 6980, - 2 + 4 ] }, { @@ -89,7 +89,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -104,7 +104,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -112,7 +112,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -127,68 +127,61 @@ "n_diagnostics_records": 0 }, "rla_training": { - "adapter_lr_multiplier": 1.0, + "adapter_lr_multiplier": 0.5, "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, "best_step": 1365, "metrics": { "train": { - "rmse": 0.514097726188904, - "mae": 0.4052426517009735, - "r2": 0.5046209692955017, - "score": -0.514097726188904 + "rmse": 0.5148489632002486, + "mae": 0.40648993849754333, + "r2": 0.5031721591949463, + "score": -0.5148489632002486 }, "val": { - "rmse": 0.5552720495549479, - "mae": 0.441690057516098, - "r2": 0.4185805916786194, - "score": -0.5552720495549479 + "rmse": 0.5553930389270275, + "mae": 0.4419495463371277, + "r2": 0.4183272123336792, + "score": -0.5553930389270275 }, "test": { - "rmse": 0.5524877260752137, - "mae": 0.447174072265625, - "r2": 0.43120694160461426, - "score": -0.5524877260752137 + "rmse": 0.5519483119271499, + "mae": 0.4472074508666992, + "r2": 0.4323171377182007, + "score": -0.5519483119271499 } }, - "time": "0:09:33.111358", + "time": "0:14:43.732239", "diversity": { "train": { - "mean_centered_corr": -0.020444847643375397, + "mean_centered_corr": -0.02016017585992813, "mean_pairwise_disagreement": NaN, - "member_std": 0.059093035757541656 + "member_std": 0.06105273962020874 }, "val": { - "mean_centered_corr": -0.020575184375047684, + "mean_centered_corr": -0.01993357017636299, "mean_pairwise_disagreement": NaN, - "member_std": 0.06087604537606239 + "member_std": 0.06221357360482216 }, "test": { - "mean_centered_corr": -0.020676201209425926, + "mean_centered_corr": -0.020221160724759102, "mean_pairwise_disagreement": NaN, - "member_std": 0.060276810079813004 + "member_std": 0.06099524348974228 } }, "chunk_size": null, "eval_batch_size": 8192, "heads": [ - 18, - 10, 11, + 10, 26, 21, - 14, + 15, + 16, 31, 19, - 11, - 16, - 26, - 10, - 11, - 15, - 21, - 29 + 13 ], - "head_selection_time": "0:00:00.514504" + "head_selection_time": "0:00:00.265073" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1/report.json index 68f3842ad..437652c9c 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/1/report.json @@ -11,7 +11,7 @@ "amp": true, "seed": 1, "head_selection": true, - "rla_adapter_lr_multiplier": 1.0, + "rla_adapter_lr_multiplier": 0.5, "rla_extra_paths_freeze_fraction": 0.0, "data": { "cache": true, @@ -26,11 +26,11 @@ "arch_type": "tabm", "k": 32, "share_training_batches": false, - "rla_rank": 2, + "rla_rank": 4, "rla_first_only": false, "rla_additive": false, "rla_init": "base_preserving", - "rla_base_preserve_noise": 0.001, + "rla_base_preserve_noise": 0.0001, "backbone": { "n_blocks": 2, "d_block": 752, @@ -45,7 +45,7 @@ "n_bins": 81 } }, - "n_parameters": 7056800, + "n_parameters": 7647904, "prediction_type": "labels", "mfb": { "enabled": false, @@ -81,7 +81,7 @@ "shape": [ 32, 6980, - 2 + 4 ] }, { @@ -89,7 +89,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -104,7 +104,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -112,7 +112,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -127,61 +127,62 @@ "n_diagnostics_records": 0 }, "rla_training": { - "adapter_lr_multiplier": 1.0, + "adapter_lr_multiplier": 0.5, "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, "best_step": 1365, "metrics": { "train": { - "rmse": 0.5148582248071162, - "mae": 0.40721189975738525, - "r2": 0.5031542778015137, - "score": -0.5148582248071162 + "rmse": 0.5146326283410019, + "mae": 0.4055343568325043, + "r2": 0.5035895109176636, + "score": -0.5146326283410019 }, "val": { - "rmse": 0.55584140086384, - "mae": 0.4416872560977936, - "r2": 0.4173876643180847, - "score": -0.55584140086384 + "rmse": 0.5557605143473513, + "mae": 0.4405376613140106, + "r2": 0.4175572395324707, + "score": -0.5557605143473513 }, "test": { - "rmse": 0.5514378793216401, - "mae": 0.44598153233528137, - "r2": 0.433366596698761, - "score": -0.5514378793216401 + "rmse": 0.5514412571081766, + "mae": 0.4451921284198761, + "r2": 0.43335968255996704, + "score": -0.5514412571081766 } }, - "time": "0:09:34.536942", + "time": "0:14:46.412006", "diversity": { "train": { - "mean_centered_corr": -0.020366203039884567, + "mean_centered_corr": -0.020146815106272697, "mean_pairwise_disagreement": NaN, - "member_std": 0.05432252958416939 + "member_std": 0.061014287173748016 }, "val": { - "mean_centered_corr": -0.020544419065117836, + "mean_centered_corr": -0.020077988505363464, "mean_pairwise_disagreement": NaN, - "member_std": 0.054353535175323486 + "member_std": 0.06329856067895889 }, "test": { - "mean_centered_corr": -0.02033921517431736, + "mean_centered_corr": -0.01972520351409912, "mean_pairwise_disagreement": NaN, - "member_std": 0.05365084111690521 + "member_std": 0.06241019815206528 } }, "chunk_size": null, "eval_batch_size": 8192, "heads": [ 5, - 24, + 12, 30, + 22, 29, - 25, - 21, - 1, + 18, 31, - 9 + 25, + 4, + 14 ], - "head_selection_time": "0:00:00.253830" + "head_selection_time": "0:00:00.290840" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2/report.json index ff59a7559..c3c6e1a67 100644 --- a/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2/report.json +++ b/tabm_integrated/paper/exp/integrated/final/delivery-eta/best_rla_only-greedy-heads-evaluation/2/report.json @@ -11,7 +11,7 @@ "amp": true, "seed": 2, "head_selection": true, - "rla_adapter_lr_multiplier": 1.0, + "rla_adapter_lr_multiplier": 0.5, "rla_extra_paths_freeze_fraction": 0.0, "data": { "cache": true, @@ -26,11 +26,11 @@ "arch_type": "tabm", "k": 32, "share_training_batches": false, - "rla_rank": 2, + "rla_rank": 4, "rla_first_only": false, "rla_additive": false, "rla_init": "base_preserving", - "rla_base_preserve_noise": 0.001, + "rla_base_preserve_noise": 0.0001, "backbone": { "n_blocks": 2, "d_block": 752, @@ -45,7 +45,7 @@ "n_bins": 81 } }, - "n_parameters": 7056800, + "n_parameters": 7647904, "prediction_type": "labels", "mfb": { "enabled": false, @@ -81,7 +81,7 @@ "shape": [ 32, 6980, - 2 + 4 ] }, { @@ -89,7 +89,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -104,7 +104,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -112,7 +112,7 @@ "shape": [ 32, 752, - 2 + 4 ] }, { @@ -127,63 +127,67 @@ "n_diagnostics_records": 0 }, "rla_training": { - "adapter_lr_multiplier": 1.0, + "adapter_lr_multiplier": 0.5, "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 1365, + "best_step": 1911, "metrics": { "train": { - "rmse": 0.5130983639400511, - "mae": 0.40277671813964844, - "r2": 0.5065450072288513, - "score": -0.5130983639400511 + "rmse": 0.5091281866287168, + "mae": 0.40107253193855286, + "r2": 0.5141518115997314, + "score": -0.5091281866287168 }, "val": { - "rmse": 0.555843679564389, - "mae": 0.4404619336128235, - "r2": 0.4173828959465027, - "score": -0.555843679564389 + "rmse": 0.5554844141815782, + "mae": 0.4408845603466034, + "r2": 0.41813576221466064, + "score": -0.5554844141815782 }, "test": { - "rmse": 0.5502940789713319, - "mae": 0.4434540569782257, - "r2": 0.4357147812843323, - "score": -0.5502940789713319 + "rmse": 0.5520847398512095, + "mae": 0.44570744037628174, + "r2": 0.4320363998413086, + "score": -0.5520847398512095 } }, - "time": "0:09:31.116943", + "time": "0:16:06.202820", "diversity": { "train": { - "mean_centered_corr": -0.020301729440689087, + "mean_centered_corr": -0.020242199301719666, "mean_pairwise_disagreement": NaN, - "member_std": 0.05993633717298508 + "member_std": 0.0667627602815628 }, "val": { - "mean_centered_corr": -0.020580198615789413, + "mean_centered_corr": -0.02033313177525997, "mean_pairwise_disagreement": NaN, - "member_std": 0.06198854371905327 + "member_std": 0.06825815886259079 }, "test": { - "mean_centered_corr": -0.02039889432489872, + "mean_centered_corr": -0.020218119025230408, "mean_pairwise_disagreement": NaN, - "member_std": 0.05979244410991669 + "member_std": 0.06682927906513214 } }, "chunk_size": null, "eval_batch_size": 8192, "heads": [ + 5, + 14, + 23, 13, - 4, - 3, - 18, 0, - 14, - 25, + 24, + 7, + 16, 29, - 22, + 8, 6, - 0 + 14, + 27, + 7, + 18 ], - "head_selection_time": "0:00:00.326608" + "head_selection_time": "0:00:00.488015" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/manifest_delivery_rescue_combined.txt b/tabm_integrated/paper/exp/integrated/manifest_delivery_rescue_combined.txt new file mode 100644 index 000000000..c798e4296 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/manifest_delivery_rescue_combined.txt @@ -0,0 +1,4 @@ +exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0.toml +exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0.toml +exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0.toml +exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0.toml diff --git a/tabm_integrated/paper/exp/integrated/manifest_delivery_rescue_final.txt b/tabm_integrated/paper/exp/integrated/manifest_delivery_rescue_final.txt new file mode 100644 index 000000000..f52fdce08 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/manifest_delivery_rescue_final.txt @@ -0,0 +1,15 @@ +exp/integrated/final/delivery-eta/best_rla_only-evaluation/0.toml +exp/integrated/final/delivery-eta/best_rla_only-evaluation/1.toml +exp/integrated/final/delivery-eta/best_rla_only-evaluation/2.toml +exp/integrated/final/delivery-eta/best_esam_only-evaluation/0.toml +exp/integrated/final/delivery-eta/best_esam_only-evaluation/1.toml +exp/integrated/final/delivery-eta/best_esam_only-evaluation/2.toml +exp/integrated/final/delivery-eta/best_mfb_only-evaluation/0.toml +exp/integrated/final/delivery-eta/best_mfb_only-evaluation/1.toml +exp/integrated/final/delivery-eta/best_mfb_only-evaluation/2.toml +exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/0.toml +exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/1.toml +exp/integrated/final/delivery-eta/best_cf_fisd_only-evaluation/2.toml +exp/integrated/final/delivery-eta/best_combined-evaluation/0.toml +exp/integrated/final/delivery-eta/best_combined-evaluation/1.toml +exp/integrated/final/delivery-eta/best_combined-evaluation/2.toml diff --git a/tabm_integrated/paper/exp/integrated/manifest_delivery_rescue_individual.txt b/tabm_integrated/paper/exp/integrated/manifest_delivery_rescue_individual.txt new file mode 100644 index 000000000..bafc9df84 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/manifest_delivery_rescue_individual.txt @@ -0,0 +1,62 @@ +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0.toml +exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0.toml diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7b026e464 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-best-head-evaluation/0/report.json @@ -0,0 +1,683 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/_esam_diagnostics/deliv_comb_esam_cf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5056157353876229, + "mae": 0.3998609483242035, + "r2": 0.5208324193954468, + "score": -0.5056157353876229 + }, + "val": { + "rmse": 0.5596782586864594, + "mae": 0.44516822695732117, + "r2": 0.4093165993690491, + "score": -0.5596782586864594 + }, + "test": { + "rmse": 0.5565908577070521, + "mae": 0.45004433393478394, + "r2": 0.42272716760635376, + "score": -0.5565908577070521 + } + }, + "time": "0:14:24.819984", + "diversity": { + "train": { + "mean_centered_corr": -0.020204994827508926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07431785017251968 + }, + "val": { + "mean_centered_corr": -0.020286401733756065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07910650968551636 + }, + "test": { + "mean_centered_corr": -0.02015332505106926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07623301446437836 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.023606" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0.toml new file mode 100644 index 000000000..a0e3cdaa7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0.toml @@ -0,0 +1,161 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = false +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 8 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0/report.json new file mode 100644 index 000000000..f3330ce7e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/0/report.json @@ -0,0 +1,679 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/_esam_diagnostics/deliv_comb_esam_cf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.4975037099949126, + "mae": 0.39285650849342346, + "r2": 0.5360844135284424, + "score": -0.4975037099949126 + }, + "val": { + "rmse": 0.5558269510040955, + "mae": 0.44211816787719727, + "r2": 0.4174179434776306, + "score": -0.5558269510040955 + }, + "test": { + "rmse": 0.5527269612276788, + "mae": 0.44693246483802795, + "r2": 0.4307142496109009, + "score": -0.5527269612276788 + } + }, + "time": "0:14:24.819984", + "diversity": { + "train": { + "mean_centered_corr": -0.020204994827508926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07431785017251968 + }, + "val": { + "mean_centered_corr": -0.020286401733756065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07910650968551636 + }, + "test": { + "mean_centered_corr": -0.02015332505106926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07623301446437836 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4367ef428 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-greedy-heads-evaluation/0/report.json @@ -0,0 +1,706 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_esam_cf-evaluation/_esam_diagnostics/deliv_comb_esam_cf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.4981544378477528, + "mae": 0.3936895430088043, + "r2": 0.5348700881004333, + "score": -0.4981544378477528 + }, + "val": { + "rmse": 0.555452544543325, + "mae": 0.44208067655563354, + "r2": 0.4182025194168091, + "score": -0.555452544543325 + }, + "test": { + "rmse": 0.5532771704027001, + "mae": 0.44795727729797363, + "r2": 0.42958033084869385, + "score": -0.5532771704027001 + } + }, + "time": "0:14:24.819984", + "diversity": { + "train": { + "mean_centered_corr": -0.020204994827508926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07431785017251968 + }, + "val": { + "mean_centered_corr": -0.020286401733756065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07910650968551636 + }, + "test": { + "mean_centered_corr": -0.02015332505106926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07623301446437836 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 24, + 31, + 15, + 7, + 13, + 8, + 1, + 27, + 26, + 12, + 10, + 28, + 19, + 2, + 31, + 7, + 13, + 15, + 5, + 8, + 21, + 1, + 4, + 31 + ], + "head_selection_time": "0:00:00.885548" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-best-head-evaluation/0/report.json new file mode 100644 index 000000000..115bc741e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-best-head-evaluation/0/report.json @@ -0,0 +1,633 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5175918577279733, + "mae": 0.40775081515312195, + "r2": 0.4978642463684082, + "score": -0.5175918577279733 + }, + "val": { + "rmse": 0.5572652691682626, + "mae": 0.44076380133628845, + "r2": 0.4143989086151123, + "score": -0.5572652691682626 + }, + "test": { + "rmse": 0.5524207799310179, + "mae": 0.4452172815799713, + "r2": 0.4313448667526245, + "score": -0.5524207799310179 + } + }, + "time": "0:15:04.586964", + "diversity": { + "train": { + "mean_centered_corr": -0.02016017585992813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06105273962020874 + }, + "val": { + "mean_centered_corr": -0.01993357017636299, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06221357360482216 + }, + "test": { + "mean_centered_corr": -0.020221160724759102, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06099524348974228 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 11 + ], + "head_selection_time": "0:00:00.023192" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0.toml new file mode 100644 index 000000000..a8cfa8c75 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0.toml @@ -0,0 +1,158 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0/report.json new file mode 100644 index 000000000..aed8e342a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0/report.json @@ -0,0 +1,629 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5150130136938161, + "mae": 0.40628108382225037, + "r2": 0.5028554797172546, + "score": -0.5150130136938161 + }, + "val": { + "rmse": 0.5559356776481553, + "mae": 0.441616415977478, + "r2": 0.4171900153160095, + "score": -0.5559356776481553 + }, + "test": { + "rmse": 0.55135486033622, + "mae": 0.445721298456192, + "r2": 0.43353718519210815, + "score": -0.55135486033622 + } + }, + "time": "0:15:04.586964", + "diversity": { + "train": { + "mean_centered_corr": -0.02016017585992813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06105273962020874 + }, + "val": { + "mean_centered_corr": -0.01993357017636299, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06221357360482216 + }, + "test": { + "mean_centered_corr": -0.020221160724759102, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06099524348974228 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..e99c8f244 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-greedy-heads-evaluation/0/report.json @@ -0,0 +1,641 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5148489632002486, + "mae": 0.40648993849754333, + "r2": 0.5031721591949463, + "score": -0.5148489632002486 + }, + "val": { + "rmse": 0.5553930389270275, + "mae": 0.4419495463371277, + "r2": 0.4183272123336792, + "score": -0.5553930389270275 + }, + "test": { + "rmse": 0.5519483119271499, + "mae": 0.4472074508666992, + "r2": 0.4323171377182007, + "score": -0.5519483119271499 + } + }, + "time": "0:15:04.586964", + "diversity": { + "train": { + "mean_centered_corr": -0.02016017585992813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06105273962020874 + }, + "val": { + "mean_centered_corr": -0.01993357017636299, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06221357360482216 + }, + "test": { + "mean_centered_corr": -0.020221160724759102, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06099524348974228 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 11, + 10, + 26, + 21, + 15, + 16, + 31, + 19, + 13 + ], + "head_selection_time": "0:00:00.258661" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ad5047df6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-best-head-evaluation/0/report.json @@ -0,0 +1,240 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/_esam_diagnostics/deliv_comb_rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5099278042219015, + "mae": 0.4058413803577423, + "r2": 0.5126245617866516, + "score": -0.5099278042219015 + }, + "val": { + "rmse": 0.5591176650506209, + "mae": 0.44280362129211426, + "r2": 0.41049933433532715, + "score": -0.5591176650506209 + }, + "test": { + "rmse": 0.5583917212440949, + "mae": 0.4517841041088104, + "r2": 0.41898560523986816, + "score": -0.5583917212440949 + } + }, + "time": "0:30:12.293206", + "diversity": { + "train": { + "mean_centered_corr": -0.020068299025297165, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07235845923423767 + }, + "val": { + "mean_centered_corr": -0.019968286156654358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0755666196346283 + }, + "test": { + "mean_centered_corr": -0.020015086978673935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07387885451316833 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023391" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0.toml new file mode 100644 index 000000000..178bf3917 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = false +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 8 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0/report.json new file mode 100644 index 000000000..aa1fff617 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0/report.json @@ -0,0 +1,236 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/_esam_diagnostics/deliv_comb_rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5014604161558279, + "mae": 0.39543983340263367, + "r2": 0.5286759734153748, + "score": -0.5014604161558279 + }, + "val": { + "rmse": 0.5555664137600386, + "mae": 0.4397052228450775, + "r2": 0.41796398162841797, + "score": -0.5555664137600386 + }, + "test": { + "rmse": 0.5512279565776417, + "mae": 0.444046288728714, + "r2": 0.4337978959083557, + "score": -0.5512279565776417 + } + }, + "time": "0:30:12.293206", + "diversity": { + "train": { + "mean_centered_corr": -0.020068299025297165, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07235845923423767 + }, + "val": { + "mean_centered_corr": -0.019968286156654358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0755666196346283 + }, + "test": { + "mean_centered_corr": -0.020015086978673935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07387885451316833 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2bc4c88e2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-greedy-heads-evaluation/0/report.json @@ -0,0 +1,252 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/_esam_diagnostics/deliv_comb_rla_esam-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5013239141557576, + "mae": 0.3959278166294098, + "r2": 0.528932511806488, + "score": -0.5013239141557576 + }, + "val": { + "rmse": 0.5550917643214267, + "mae": 0.4404295086860657, + "r2": 0.41895806789398193, + "score": -0.5550917643214267 + }, + "test": { + "rmse": 0.5526470209655567, + "mae": 0.4466344714164734, + "r2": 0.4308789372444153, + "score": -0.5526470209655567 + } + }, + "time": "0:30:12.293206", + "diversity": { + "train": { + "mean_centered_corr": -0.020068299025297165, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07235845923423767 + }, + "val": { + "mean_centered_corr": -0.019968286156654358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0755666196346283 + }, + "test": { + "mean_centered_corr": -0.020015086978673935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07387885451316833 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 20, + 21, + 11, + 8, + 12, + 7, + 23, + 19, + 15, + 31, + 28, + 22 + ], + "head_selection_time": "0:00:00.393915" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7e536755e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/_esam_diagnostics/deliv_comb_rla_esam_cf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5099278042219015, + "mae": 0.4058413803577423, + "r2": 0.5126245617866516, + "score": -0.5099278042219015 + }, + "val": { + "rmse": 0.5591176650506209, + "mae": 0.44280362129211426, + "r2": 0.41049933433532715, + "score": -0.5591176650506209 + }, + "test": { + "rmse": 0.5583917212440949, + "mae": 0.4517841041088104, + "r2": 0.41898560523986816, + "score": -0.5583917212440949 + } + }, + "time": "0:31:32.729092", + "diversity": { + "train": { + "mean_centered_corr": -0.020068299025297165, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07235845923423767 + }, + "val": { + "mean_centered_corr": -0.019968286156654358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0755666196346283 + }, + "test": { + "mean_centered_corr": -0.020015086978673935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07387885451316833 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023934" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0.toml new file mode 100644 index 000000000..33ce5f9fe --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0.toml @@ -0,0 +1,168 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = false +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 8 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0/report.json new file mode 100644 index 000000000..5b29fb924 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/_esam_diagnostics/deliv_comb_rla_esam_cf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5014604161558279, + "mae": 0.39543983340263367, + "r2": 0.5286759734153748, + "score": -0.5014604161558279 + }, + "val": { + "rmse": 0.5555664137600386, + "mae": 0.4397052228450775, + "r2": 0.41796398162841797, + "score": -0.5555664137600386 + }, + "test": { + "rmse": 0.5512279565776417, + "mae": 0.444046288728714, + "r2": 0.4337978959083557, + "score": -0.5512279565776417 + } + }, + "time": "0:31:32.729092", + "diversity": { + "train": { + "mean_centered_corr": -0.020068299025297165, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07235845923423767 + }, + "val": { + "mean_centered_corr": -0.019968286156654358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0755666196346283 + }, + "test": { + "mean_centered_corr": -0.020015086978673935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07387885451316833 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..da59ef9ea --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-greedy-heads-evaluation/0/report.json @@ -0,0 +1,706 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/_esam_diagnostics/deliv_comb_rla_esam_cf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5013239141557576, + "mae": 0.3959278166294098, + "r2": 0.528932511806488, + "score": -0.5013239141557576 + }, + "val": { + "rmse": 0.5550917643214267, + "mae": 0.4404295086860657, + "r2": 0.41895806789398193, + "score": -0.5550917643214267 + }, + "test": { + "rmse": 0.5526470209655567, + "mae": 0.4466344714164734, + "r2": 0.4308789372444153, + "score": -0.5526470209655567 + } + }, + "time": "0:31:32.729092", + "diversity": { + "train": { + "mean_centered_corr": -0.020068299025297165, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07235845923423767 + }, + "val": { + "mean_centered_corr": -0.019968286156654358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0755666196346283 + }, + "test": { + "mean_centered_corr": -0.020015086978673935, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07387885451316833 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 20, + 21, + 11, + 8, + 12, + 7, + 23, + 19, + 15, + 31, + 28, + 22 + ], + "head_selection_time": "0:00:00.396932" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-best-head-evaluation/0/report.json new file mode 100644 index 000000000..011d702b0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-best-head-evaluation/0/report.json @@ -0,0 +1,622 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:50.667909", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023443" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0.toml new file mode 100644 index 000000000..e7d7fb4f3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0.toml @@ -0,0 +1,151 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0/report.json new file mode 100644 index 000000000..6337eebc5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0/report.json @@ -0,0 +1,618 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:50.667909", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9a27827e8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-greedy-heads-evaluation/0/report.json @@ -0,0 +1,635 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:50.667909", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.439011" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-best-head-evaluation/0/report.json new file mode 100644 index 000000000..2869e0596 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-best-head-evaluation/0/report.json @@ -0,0 +1,454 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "softmax", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "softmax" + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "softmax", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "softmax" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:32.728471", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023671" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0.toml new file mode 100644 index 000000000..92fc7d60c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.005 +variant = "softmax" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "softmax" diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0/report.json new file mode 100644 index 000000000..dc21703c3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-evaluation/0/report.json @@ -0,0 +1,450 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "softmax", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "softmax" + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "softmax", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "softmax" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:32.728471", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1aac6f0ff --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_softmax-greedy-heads-evaluation/0/report.json @@ -0,0 +1,467 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "softmax", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "softmax" + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "softmax", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "softmax" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:32.728471", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.426444" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4e75654ba --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-best-head-evaluation/0/report.json @@ -0,0 +1,622 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:47.591509", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023656" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0.toml new file mode 100644 index 000000000..853f0f9cb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0.toml @@ -0,0 +1,151 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0/report.json new file mode 100644 index 000000000..9e9a5a4c6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-evaluation/0/report.json @@ -0,0 +1,618 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:47.591509", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..76d9893e9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_consensus_raw-greedy-heads-evaluation/0/report.json @@ -0,0 +1,635 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:47.591509", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.432736" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b002af603 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-best-head-evaluation/0/report.json @@ -0,0 +1,454 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "softmax", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "softmax" + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "softmax", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "softmax" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:33.670332", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023709" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0.toml new file mode 100644 index 000000000..89f0c2869 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.01 +variant = "softmax" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "softmax" diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0/report.json new file mode 100644 index 000000000..f18168d9c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-evaluation/0/report.json @@ -0,0 +1,450 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "softmax", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "softmax" + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "softmax", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "softmax" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:33.670332", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..fcc8f2184 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p01_softmax-greedy-heads-evaluation/0/report.json @@ -0,0 +1,467 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "softmax", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "softmax" + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "softmax", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "softmax" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:33.670332", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.432860" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3a8df65db --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-best-head-evaluation/0/report.json @@ -0,0 +1,622 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:49.125262", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023170" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0.toml new file mode 100644 index 000000000..a1a0983ff --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0.toml @@ -0,0 +1,151 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "consensus_raw" + +[cf_fisd.member_groups] +xgb = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +lgbm = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] +cat = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0/report.json new file mode 100644 index 000000000..75a48e37f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-evaluation/0/report.json @@ -0,0 +1,618 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:49.125262", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..554cf3c71 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_consensus_raw-greedy-heads-evaluation/0/report.json @@ -0,0 +1,635 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "consensus_raw", + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + } + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "lgbm": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "cat": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "consensus_raw" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:49.125262", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.432808" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8f7a20eb4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-best-head-evaluation/0/report.json @@ -0,0 +1,454 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "softmax", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "softmax" + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "softmax", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "softmax" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:31.619857", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023385" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0.toml new file mode 100644 index 000000000..2005313ed --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0.toml @@ -0,0 +1,47 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 + +[cf_fisd] +lambda = 0.02 +variant = "softmax" +dataset_name = "delivery-eta" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] +start_epoch = 16 +start_fraction = 0.5 +mode = "softmax" diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0/report.json new file mode 100644 index 000000000..ba314e909 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-evaluation/0/report.json @@ -0,0 +1,450 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "softmax", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "softmax" + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "softmax", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "softmax" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:31.619857", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3b5fe62a8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p02_softmax-greedy-heads-evaluation/0/report.json @@ -0,0 +1,467 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "softmax", + "dataset_name": "delivery-eta", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "start_epoch": 16, + "start_fraction": 0.5, + "mode": "softmax" + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "softmax", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 2, + 2 + ], + "n_features": 220, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/delivery-eta", + "start_epoch": 16, + "mode": "softmax" + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:31.619857", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.443035" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-best-head-evaluation/0/report.json new file mode 100644 index 000000000..36228be09 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:11:54.514683", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023259" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0.toml new file mode 100644 index 000000000..ce06a82a3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = false +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 8 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0/report.json new file mode 100644 index 000000000..4178bd75e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:11:54.514683", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7daaef5ec --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-greedy-heads-evaluation/0/report.json @@ -0,0 +1,242 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p25_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:11:54.514683", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.437296" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-best-head-evaluation/0/report.json new file mode 100644 index 000000000..21e979ca4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-best-head-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224530864115, + "mae": 0.40589213371276855, + "r2": 0.5101466178894043, + "score": -0.5112224530864115 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:10:58.902882", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023427" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0.toml new file mode 100644 index 000000000..e552d66e3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 8 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0/report.json new file mode 100644 index 000000000..cfccf217f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558750174513605, + "mae": 0.4398687481880188, + "r2": 0.41731715202331543, + "score": -0.5558750174513605 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:10:58.902882", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..07089b738 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p25_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.50422871052893, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.50422871052893 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:10:58.902882", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.433917" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1e121ca71 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:09:36.298262", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023756" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0.toml new file mode 100644 index 000000000..81af33fff --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = false +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 16 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0/report.json new file mode 100644 index 000000000..b05705786 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:09:36.298262", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d1f58753b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-greedy-heads-evaluation/0/report.json @@ -0,0 +1,242 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p5_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:09:36.298262", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.434693" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b3701dc27 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-best-head-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224530864115, + "mae": 0.40589213371276855, + "r2": 0.5101466178894043, + "score": -0.5112224530864115 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:09:06.499354", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023446" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0.toml new file mode 100644 index 000000000..66221114c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 16 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0/report.json new file mode 100644 index 000000000..7e408ea6d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558750174513605, + "mae": 0.4398687481880188, + "r2": 0.41731715202331543, + "score": -0.5558750174513605 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:09:06.499354", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7de1edf8d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p00025_sf0p5_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.50422871052893, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.50422871052893 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:09:06.499354", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.437358" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-best-head-evaluation/0/report.json new file mode 100644 index 000000000..06abf0f83 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:11:58.210461", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023576" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0.toml new file mode 100644 index 000000000..f6ebc3b63 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = false +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 8 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0/report.json new file mode 100644 index 000000000..82de312dc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:11:58.210461", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..dcac0964e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-greedy-heads-evaluation/0/report.json @@ -0,0 +1,242 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p25_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:11:58.210461", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.429491" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-best-head-evaluation/0/report.json new file mode 100644 index 000000000..596adaf42 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-best-head-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224530864115, + "mae": 0.40589213371276855, + "r2": 0.5101466178894043, + "score": -0.5112224530864115 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:11:00.793096", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023362" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0.toml new file mode 100644 index 000000000..067d1c90d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 8 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0/report.json new file mode 100644 index 000000000..953f014db --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558750174513605, + "mae": 0.4398687481880188, + "r2": 0.41731715202331543, + "score": -0.5558750174513605 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:11:00.793096", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..6acf4b8e9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p25_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.50422871052893, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.50422871052893 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:11:00.793096", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.431835" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3fdfbbd30 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:09:36.252294", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023761" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0.toml new file mode 100644 index 000000000..6ac8d4057 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = false +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 16 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0/report.json new file mode 100644 index 000000000..be11ffae9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:09:36.252294", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0a18f0c54 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-greedy-heads-evaluation/0/report.json @@ -0,0 +1,242 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p5_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:09:36.252294", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.429945" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-best-head-evaluation/0/report.json new file mode 100644 index 000000000..45c3e689f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-best-head-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224530864115, + "mae": 0.40589213371276855, + "r2": 0.5101466178894043, + "score": -0.5112224530864115 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:09:09.025241", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023020" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0.toml new file mode 100644 index 000000000..867546608 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 16 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0/report.json new file mode 100644 index 000000000..9aea75d97 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558750174513605, + "mae": 0.4398687481880188, + "r2": 0.41731715202331543, + "score": -0.5558750174513605 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:09:09.025241", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..690c5e88a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p0005_sf0p5_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.50422871052893, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.50422871052893 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:09:09.025241", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.422502" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3e99b717d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p25_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.5056157353876229, + "mae": 0.3998609483242035, + "r2": 0.5208324193954468, + "score": -0.5056157353876229 + }, + "val": { + "rmse": 0.5596782586864594, + "mae": 0.44516822695732117, + "r2": 0.4093165993690491, + "score": -0.5596782586864594 + }, + "test": { + "rmse": 0.5565908577070521, + "mae": 0.45004433393478394, + "r2": 0.42272716760635376, + "score": -0.5565908577070521 + } + }, + "time": "0:13:05.256760", + "diversity": { + "train": { + "mean_centered_corr": -0.020204994827508926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07431785017251968 + }, + "val": { + "mean_centered_corr": -0.020286401733756065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07910650968551636 + }, + "test": { + "mean_centered_corr": -0.02015332505106926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07623301446437836 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.023424" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0.toml new file mode 100644 index 000000000..474196b1e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = false +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 8 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0/report.json new file mode 100644 index 000000000..923d84058 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p25_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.4975037099949126, + "mae": 0.39285650849342346, + "r2": 0.5360844135284424, + "score": -0.4975037099949126 + }, + "val": { + "rmse": 0.5558269510040955, + "mae": 0.44211816787719727, + "r2": 0.4174179434776306, + "score": -0.5558269510040955 + }, + "test": { + "rmse": 0.5527269612276788, + "mae": 0.44693246483802795, + "r2": 0.4307142496109009, + "score": -0.5527269612276788 + } + }, + "time": "0:13:05.256760", + "diversity": { + "train": { + "mean_centered_corr": -0.020204994827508926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07431785017251968 + }, + "val": { + "mean_centered_corr": -0.020286401733756065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07910650968551636 + }, + "test": { + "mean_centered_corr": -0.02015332505106926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07623301446437836 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f8a336e9b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-greedy-heads-evaluation/0/report.json @@ -0,0 +1,252 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p25_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2730, + "metrics": { + "train": { + "rmse": 0.4981544378477528, + "mae": 0.3936895430088043, + "r2": 0.5348700881004333, + "score": -0.4981544378477528 + }, + "val": { + "rmse": 0.555452544543325, + "mae": 0.44208067655563354, + "r2": 0.4182025194168091, + "score": -0.555452544543325 + }, + "test": { + "rmse": 0.5532771704027001, + "mae": 0.44795727729797363, + "r2": 0.42958033084869385, + "score": -0.5532771704027001 + } + }, + "time": "0:13:05.256760", + "diversity": { + "train": { + "mean_centered_corr": -0.020204994827508926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07431785017251968 + }, + "val": { + "mean_centered_corr": -0.020286401733756065, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07910650968551636 + }, + "test": { + "mean_centered_corr": -0.02015332505106926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07623301446437836 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 24, + 31, + 15, + 7, + 13, + 8, + 1, + 27, + 26, + 12, + 10, + 28, + 19, + 2, + 31, + 7, + 13, + 15, + 5, + 8, + 21, + 1, + 4, + 31 + ], + "head_selection_time": "0:00:00.922852" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ced6ae76a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-best-head-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p25_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224530864115, + "mae": 0.40589213371276855, + "r2": 0.5101466178894043, + "score": -0.5112224530864115 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:10:59.649752", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023376" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0.toml new file mode 100644 index 000000000..d3c1f452d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 8 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0/report.json new file mode 100644 index 000000000..835158685 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p25_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558750174513605, + "mae": 0.4398687481880188, + "r2": 0.41731715202331543, + "score": -0.5558750174513605 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:10:59.649752", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..998291c51 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 8, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p25_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.50422871052893, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.50422871052893 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:10:59.649752", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.425940" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-best-head-evaluation/0/report.json new file mode 100644 index 000000000..18e09303d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-best-head-evaluation/0/report.json @@ -0,0 +1,229 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p5_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:09:34.700424", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023611" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0.toml new file mode 100644 index 000000000..a5511d194 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = false +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 16 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0/report.json new file mode 100644 index 000000000..1bc764eaa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p5_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:09:34.700424", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..353b2b7b7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-greedy-heads-evaluation/0/report.json @@ -0,0 +1,242 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": false, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "num_module.linear0.weight", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear0.bias", + "shape": [ + 218, + 32 + ] + }, + { + "name": "num_module.linear.weight", + "shape": [ + 218, + 81, + 32 + ] + }, + { + "name": "backbone.blocks.0.0.weight", + "shape": [ + 752, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.weight", + "shape": [ + 752, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "output.weight", + "shape": [ + 32, + 752, + 1 + ] + }, + { + "name": "output.bias", + "shape": [ + 32, + 1 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adapterf-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p5_adapterf-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:09:34.700424", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.436537" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9a11da961 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-best-head-evaluation/0/report.json @@ -0,0 +1,178 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p5_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224530864115, + "mae": 0.40589213371276855, + "r2": 0.5101466178894043, + "score": -0.5112224530864115 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:09:06.259977", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023674" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0.toml new file mode 100644 index 000000000..79f334591 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0.toml @@ -0,0 +1,43 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 16 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0/report.json new file mode 100644 index 000000000..5a09ad7c7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/0/report.json @@ -0,0 +1,174 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p5_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558750174513605, + "mae": 0.4398687481880188, + "r2": 0.41731715202331543, + "score": -0.5558750174513605 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:09:06.259977", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..23466521b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 16, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p5_adaptert-evaluation/_esam_diagnostics/deliv_esam_rho0p001_sf0p5_adaptert-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.50422871052893, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.50422871052893 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:09:06.259977", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442888140678406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958992034196854 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.420549" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e12ce29d6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-best-head-evaluation/0/report.json @@ -0,0 +1,212 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5081016011029494, + "mae": 0.4003114700317383, + "r2": 0.5161091685295105, + "score": -0.5081016011029494 + }, + "val": { + "rmse": 0.5590051856389011, + "mae": 0.4407220184803009, + "r2": 0.41073644161224365, + "score": -0.5590051856389011 + }, + "test": { + "rmse": 0.5534488118080276, + "mae": 0.4457874596118927, + "r2": 0.4292263984680176, + "score": -0.5534488118080276 + } + }, + "time": "0:07:44.425919", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6 + ], + "head_selection_time": "0:00:00.024746" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0.toml new file mode 100644 index 000000000..ba5fa4110 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 +start_epoch = 16 +start_fraction = 0.5 +group_mode = "numerical_only" +categorical_handling = "no_cat_drop" + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0/report.json new file mode 100644 index 000000000..3f118c5a6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5047748974218713, + "mae": 0.39609476923942566, + "r2": 0.5224248170852661, + "score": -0.5047748974218713 + }, + "val": { + "rmse": 0.5558048331442785, + "mae": 0.43993815779685974, + "r2": 0.41746431589126587, + "score": -0.5558048331442785 + }, + "test": { + "rmse": 0.5501619198178428, + "mae": 0.44212156534194946, + "r2": 0.435985803604126, + "score": -0.5501619198178428 + } + }, + "time": "0:07:44.425919", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b43e4ade6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5051252716316714, + "mae": 0.39751043915748596, + "r2": 0.5217615962028503, + "score": -0.5051252716316714 + }, + "val": { + "rmse": 0.5553099940581016, + "mae": 0.44019240140914917, + "r2": 0.4185011386871338, + "score": -0.5553099940581016 + }, + "test": { + "rmse": 0.5510631408064661, + "mae": 0.44413861632347107, + "r2": 0.43413645029067993, + "score": -0.5510631408064661 + } + }, + "time": "0:07:44.425919", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6, + 13, + 26, + 10, + 14, + 8, + 7, + 27, + 17, + 26, + 13, + 25, + 19 + ], + "head_selection_time": "0:00:00.409868" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-best-head-evaluation/0/report.json new file mode 100644 index 000000000..2144dbac0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-best-head-evaluation/0/report.json @@ -0,0 +1,212 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5081016011029494, + "mae": 0.4003114700317383, + "r2": 0.5161091685295105, + "score": -0.5081016011029494 + }, + "val": { + "rmse": 0.5590051856389011, + "mae": 0.4407220184803009, + "r2": 0.41073644161224365, + "score": -0.5590051856389011 + }, + "test": { + "rmse": 0.5534488118080276, + "mae": 0.4457874596118927, + "r2": 0.4292263984680176, + "score": -0.5534488118080276 + } + }, + "time": "0:07:43.333698", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6 + ], + "head_selection_time": "0:00:00.023570" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0.toml new file mode 100644 index 000000000..feee7a060 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 +start_epoch = 16 +start_fraction = 0.5 +group_mode = "per_member" +categorical_handling = "no_cat_drop" + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0/report.json new file mode 100644 index 000000000..c354cbffb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-evaluation/0/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5047748974218713, + "mae": 0.39609476923942566, + "r2": 0.5224248170852661, + "score": -0.5047748974218713 + }, + "val": { + "rmse": 0.5558048331442785, + "mae": 0.43993815779685974, + "r2": 0.41746431589126587, + "score": -0.5558048331442785 + }, + "test": { + "rmse": 0.5501619198178428, + "mae": 0.44212156534194946, + "r2": 0.435985803604126, + "score": -0.5501619198178428 + } + }, + "time": "0:07:43.333698", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..bb3c18a92 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_per_member-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5051252716316714, + "mae": 0.39751043915748596, + "r2": 0.5217615962028503, + "score": -0.5051252716316714 + }, + "val": { + "rmse": 0.5553099940581016, + "mae": 0.44019240140914917, + "r2": 0.4185011386871338, + "score": -0.5553099940581016 + }, + "test": { + "rmse": 0.5510631408064661, + "mae": 0.44413861632347107, + "r2": 0.43413645029067993, + "score": -0.5510631408064661 + } + }, + "time": "0:07:43.333698", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6, + 13, + 26, + 10, + 14, + 8, + 7, + 27, + 17, + 26, + 13, + 25, + 19 + ], + "head_selection_time": "0:00:00.397135" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e90bc20e9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-best-head-evaluation/0/report.json @@ -0,0 +1,212 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5081016011029494, + "mae": 0.4003114700317383, + "r2": 0.5161091685295105, + "score": -0.5081016011029494 + }, + "val": { + "rmse": 0.5590051856389011, + "mae": 0.4407220184803009, + "r2": 0.41073644161224365, + "score": -0.5590051856389011 + }, + "test": { + "rmse": 0.5534488118080276, + "mae": 0.4457874596118927, + "r2": 0.4292263984680176, + "score": -0.5534488118080276 + } + }, + "time": "0:07:18.478027", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6 + ], + "head_selection_time": "0:00:00.023345" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0.toml new file mode 100644 index 000000000..e61f34d87 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 +start_epoch = 24 +start_fraction = 0.75 +group_mode = "numerical_only" +categorical_handling = "no_cat_drop" + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0/report.json new file mode 100644 index 000000000..2ffadf096 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-evaluation/0/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5047748974218713, + "mae": 0.39609476923942566, + "r2": 0.5224248170852661, + "score": -0.5047748974218713 + }, + "val": { + "rmse": 0.5558048331442785, + "mae": 0.43993815779685974, + "r2": 0.41746431589126587, + "score": -0.5558048331442785 + }, + "test": { + "rmse": 0.5501619198178428, + "mae": 0.44212156534194946, + "r2": 0.435985803604126, + "score": -0.5501619198178428 + } + }, + "time": "0:07:18.478027", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..05a9734ef --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_numerical_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5051252716316714, + "mae": 0.39751043915748596, + "r2": 0.5217615962028503, + "score": -0.5051252716316714 + }, + "val": { + "rmse": 0.5553099940581016, + "mae": 0.44019240140914917, + "r2": 0.4185011386871338, + "score": -0.5553099940581016 + }, + "test": { + "rmse": 0.5510631408064661, + "mae": 0.44413861632347107, + "r2": 0.43413645029067993, + "score": -0.5510631408064661 + } + }, + "time": "0:07:18.478027", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6, + 13, + 26, + 10, + 14, + 8, + 7, + 27, + 17, + 26, + 13, + 25, + 19 + ], + "head_selection_time": "0:00:00.396856" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c5f593cdf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-best-head-evaluation/0/report.json @@ -0,0 +1,212 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5081016011029494, + "mae": 0.4003114700317383, + "r2": 0.5161091685295105, + "score": -0.5081016011029494 + }, + "val": { + "rmse": 0.5590051856389011, + "mae": 0.4407220184803009, + "r2": 0.41073644161224365, + "score": -0.5590051856389011 + }, + "test": { + "rmse": 0.5534488118080276, + "mae": 0.4457874596118927, + "r2": 0.4292263984680176, + "score": -0.5534488118080276 + } + }, + "time": "0:07:19.574103", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6 + ], + "head_selection_time": "0:00:00.023618" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0.toml new file mode 100644 index 000000000..636b0362e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 +start_epoch = 24 +start_fraction = 0.75 +group_mode = "per_member" +categorical_handling = "no_cat_drop" + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0/report.json new file mode 100644 index 000000000..25addf8bd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-evaluation/0/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5047748974218713, + "mae": 0.39609476923942566, + "r2": 0.5224248170852661, + "score": -0.5047748974218713 + }, + "val": { + "rmse": 0.5558048331442785, + "mae": 0.43993815779685974, + "r2": 0.41746431589126587, + "score": -0.5558048331442785 + }, + "test": { + "rmse": 0.5501619198178428, + "mae": 0.44212156534194946, + "r2": 0.435985803604126, + "score": -0.5501619198178428 + } + }, + "time": "0:07:19.574103", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7683a0567 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p75_per_member-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9772727489471436, + "min_features_per_member": 210.0, + "mean_features_per_member": 215.0, + "max_features_per_member": 220.0, + "min_members_per_feature": 29.0, + "mean_members_per_feature": 31.272727966308594, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.955511449861677, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5051252716316714, + "mae": 0.39751043915748596, + "r2": 0.5217615962028503, + "score": -0.5051252716316714 + }, + "val": { + "rmse": 0.5553099940581016, + "mae": 0.44019240140914917, + "r2": 0.4185011386871338, + "score": -0.5553099940581016 + }, + "test": { + "rmse": 0.5510631408064661, + "mae": 0.44413861632347107, + "r2": 0.43413645029067993, + "score": -0.5510631408064661 + } + }, + "time": "0:07:19.574103", + "diversity": { + "train": { + "mean_centered_corr": -0.02009296789765358, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07790226489305496 + }, + "val": { + "mean_centered_corr": -0.02050519548356533, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07800902426242828 + }, + "test": { + "mean_centered_corr": -0.02043369971215725, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07650945335626602 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 6, + 13, + 26, + 10, + 14, + 8, + 7, + 27, + 17, + 26, + 13, + 25, + 19 + ], + "head_selection_time": "0:00:00.395432" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bde2c36a5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-best-head-evaluation/0/report.json @@ -0,0 +1,212 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5111845008565297, + "mae": 0.4058074951171875, + "r2": 0.5102193355560303, + "score": -0.5111845008565297 + }, + "val": { + "rmse": 0.5584120821811833, + "mae": 0.44400346279144287, + "r2": 0.41198623180389404, + "score": -0.5584120821811833 + }, + "test": { + "rmse": 0.5559629363941286, + "mae": 0.4507065415382385, + "r2": 0.4240289330482483, + "score": -0.5559629363941286 + } + }, + "time": "0:07:44.999332", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023353" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0.toml new file mode 100644 index 000000000..8b577989a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.99 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 +start_epoch = 16 +start_fraction = 0.5 +group_mode = "numerical_only" +categorical_handling = "no_cat_drop" + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0/report.json new file mode 100644 index 000000000..235a79e5d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-evaluation/0/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042872799552254, + "mae": 0.39527493715286255, + "r2": 0.523347020149231, + "score": -0.5042872799552254 + }, + "val": { + "rmse": 0.5558106240871751, + "mae": 0.4399650990962982, + "r2": 0.4174521565437317, + "score": -0.5558106240871751 + }, + "test": { + "rmse": 0.5498568034160569, + "mae": 0.44145897030830383, + "r2": 0.43661123514175415, + "score": -0.5498568034160569 + } + }, + "time": "0:07:44.999332", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..805f8d13c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_numerical_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5048514675541059, + "mae": 0.397201806306839, + "r2": 0.5222799181938171, + "score": -0.5048514675541059 + }, + "val": { + "rmse": 0.5553168366734594, + "mae": 0.4400327205657959, + "r2": 0.4184867739677429, + "score": -0.5553168366734594 + }, + "test": { + "rmse": 0.5509050649441617, + "mae": 0.44378286600112915, + "r2": 0.4344610571861267, + "score": -0.5509050649441617 + } + }, + "time": "0:07:44.999332", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 14, + 10, + 26, + 27, + 7, + 6, + 29, + 8, + 15, + 17, + 6 + ], + "head_selection_time": "0:00:00.394414" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1886d86c0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-best-head-evaluation/0/report.json @@ -0,0 +1,212 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5111845008565297, + "mae": 0.4058074951171875, + "r2": 0.5102193355560303, + "score": -0.5111845008565297 + }, + "val": { + "rmse": 0.5584120821811833, + "mae": 0.44400346279144287, + "r2": 0.41198623180389404, + "score": -0.5584120821811833 + }, + "test": { + "rmse": 0.5559629363941286, + "mae": 0.4507065415382385, + "r2": 0.4240289330482483, + "score": -0.5559629363941286 + } + }, + "time": "0:07:44.772554", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023456" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0.toml new file mode 100644 index 000000000..08a257a0b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.99 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 +start_epoch = 16 +start_fraction = 0.5 +group_mode = "per_member" +categorical_handling = "no_cat_drop" + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0/report.json new file mode 100644 index 000000000..d0a942f0b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-evaluation/0/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042872799552254, + "mae": 0.39527493715286255, + "r2": 0.523347020149231, + "score": -0.5042872799552254 + }, + "val": { + "rmse": 0.5558106240871751, + "mae": 0.4399650990962982, + "r2": 0.4174521565437317, + "score": -0.5558106240871751 + }, + "test": { + "rmse": 0.5498568034160569, + "mae": 0.44145897030830383, + "r2": 0.43661123514175415, + "score": -0.5498568034160569 + } + }, + "time": "0:07:44.772554", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..cb8c389b1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p5_per_member-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 16, + "start_fraction": 0.5, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5048514675541059, + "mae": 0.397201806306839, + "r2": 0.5222799181938171, + "score": -0.5048514675541059 + }, + "val": { + "rmse": 0.5553168366734594, + "mae": 0.4400327205657959, + "r2": 0.4184867739677429, + "score": -0.5553168366734594 + }, + "test": { + "rmse": 0.5509050649441617, + "mae": 0.44378286600112915, + "r2": 0.4344610571861267, + "score": -0.5509050649441617 + } + }, + "time": "0:07:44.772554", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 14, + 10, + 26, + 27, + 7, + 6, + 29, + 8, + 15, + 17, + 6 + ], + "head_selection_time": "0:00:00.400193" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4b7f7549e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-best-head-evaluation/0/report.json @@ -0,0 +1,212 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5111845008565297, + "mae": 0.4058074951171875, + "r2": 0.5102193355560303, + "score": -0.5111845008565297 + }, + "val": { + "rmse": 0.5584120821811833, + "mae": 0.44400346279144287, + "r2": 0.41198623180389404, + "score": -0.5584120821811833 + }, + "test": { + "rmse": 0.5559629363941286, + "mae": 0.4507065415382385, + "r2": 0.4240289330482483, + "score": -0.5559629363941286 + } + }, + "time": "0:07:21.324075", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023316" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0.toml new file mode 100644 index 000000000..13878a19d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.99 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 +start_epoch = 24 +start_fraction = 0.75 +group_mode = "numerical_only" +categorical_handling = "no_cat_drop" + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0/report.json new file mode 100644 index 000000000..501bd3a58 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-evaluation/0/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042872799552254, + "mae": 0.39527493715286255, + "r2": 0.523347020149231, + "score": -0.5042872799552254 + }, + "val": { + "rmse": 0.5558106240871751, + "mae": 0.4399650990962982, + "r2": 0.4174521565437317, + "score": -0.5558106240871751 + }, + "test": { + "rmse": 0.5498568034160569, + "mae": 0.44145897030830383, + "r2": 0.43661123514175415, + "score": -0.5498568034160569 + } + }, + "time": "0:07:21.324075", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..39b35ad63 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_numerical_only-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "numerical_only", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5048514675541059, + "mae": 0.397201806306839, + "r2": 0.5222799181938171, + "score": -0.5048514675541059 + }, + "val": { + "rmse": 0.5553168366734594, + "mae": 0.4400327205657959, + "r2": 0.4184867739677429, + "score": -0.5553168366734594 + }, + "test": { + "rmse": 0.5509050649441617, + "mae": 0.44378286600112915, + "r2": 0.4344610571861267, + "score": -0.5509050649441617 + } + }, + "time": "0:07:21.324075", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 14, + 10, + 26, + 27, + 7, + 6, + 29, + 8, + 15, + 17, + 6 + ], + "head_selection_time": "0:00:00.396168" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4fa13f71b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-best-head-evaluation/0/report.json @@ -0,0 +1,212 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5111845008565297, + "mae": 0.4058074951171875, + "r2": 0.5102193355560303, + "score": -0.5111845008565297 + }, + "val": { + "rmse": 0.5584120821811833, + "mae": 0.44400346279144287, + "r2": 0.41198623180389404, + "score": -0.5584120821811833 + }, + "test": { + "rmse": 0.5559629363941286, + "mae": 0.4507065415382385, + "r2": 0.4240289330482483, + "score": -0.5559629363941286 + } + }, + "time": "0:07:20.229776", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023284" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0.toml new file mode 100644 index 000000000..44f21d6df --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0.toml @@ -0,0 +1,50 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.99 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 +start_epoch = 24 +start_fraction = 0.75 +group_mode = "per_member" +categorical_handling = "no_cat_drop" + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0/report.json new file mode 100644 index 000000000..6dbbb8057 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-evaluation/0/report.json @@ -0,0 +1,208 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042872799552254, + "mae": 0.39527493715286255, + "r2": 0.523347020149231, + "score": -0.5042872799552254 + }, + "val": { + "rmse": 0.5558106240871751, + "mae": 0.4399650990962982, + "r2": 0.4174521565437317, + "score": -0.5558106240871751 + }, + "test": { + "rmse": 0.5498568034160569, + "mae": 0.44145897030830383, + "r2": 0.43661123514175415, + "score": -0.5498568034160569 + } + }, + "time": "0:07:20.229776", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..312a01d8a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p99_sf0p75_per_member-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.99, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0, + "start_epoch": 24, + "start_fraction": 0.75, + "group_mode": "per_member", + "categorical_handling": "no_cat_drop" + }, + "mask_stats": { + "keep_rate_actual": 0.9903408885002136, + "min_features_per_member": 213.0, + "mean_features_per_member": 217.875, + "max_features_per_member": 220.0, + "min_members_per_feature": 30.0, + "mean_members_per_feature": 31.690908432006836, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9807907427121413, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.00909090909090909, + "n_core_features": 2.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5048514675541059, + "mae": 0.397201806306839, + "r2": 0.5222799181938171, + "score": -0.5048514675541059 + }, + "val": { + "rmse": 0.5553168366734594, + "mae": 0.4400327205657959, + "r2": 0.4184867739677429, + "score": -0.5553168366734594 + }, + "test": { + "rmse": 0.5509050649441617, + "mae": 0.44378286600112915, + "r2": 0.4344610571861267, + "score": -0.5509050649441617 + } + }, + "time": "0:07:20.229776", + "diversity": { + "train": { + "mean_centered_corr": -0.02024354599416256, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07271759957075119 + }, + "val": { + "mean_centered_corr": -0.02031630650162697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07322525978088379 + }, + "test": { + "mean_centered_corr": -0.02031359262764454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07238791882991791 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 6, + 14, + 10, + 26, + 27, + 7, + 6, + 29, + 8, + 15, + 17, + 6 + ], + "head_selection_time": "0:00:00.394302" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f195dc2f3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:16.610424", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.024247" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0.toml new file mode 100644 index 000000000..437092c8a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0/report.json new file mode 100644 index 000000000..862d22692 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:16.610424", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..94b76121e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:16.610424", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.455776" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..61809e291 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:17.563806", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.025046" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..683fcc68f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..b7563688f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:17.563806", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..db5c58c9a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:17.563806", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.459154" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7ac826b41 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:15.282501", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023342" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0.toml new file mode 100644 index 000000000..fad5aa191 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0/report.json new file mode 100644 index 000000000..2eea74d43 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:15.282501", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..6f94ed1c7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:15.282501", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.438429" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..074f2755a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:16.646461", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023405" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..f7d34ae59 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..831b044e8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:16.646461", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9e9ee3780 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:16.646461", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.435497" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ee6bb5730 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:16.567442", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023072" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0.toml new file mode 100644 index 000000000..a6084c224 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0/report.json new file mode 100644 index 000000000..c0683acdb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:16.567442", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ae4079b97 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:16.567442", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.436208" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1e774b115 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:15.560400", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023828" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..99e15b430 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..4089927f2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:15.560400", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f96938ad2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:15.560400", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.438707" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5081de80f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:16.936182", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023175" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0.toml new file mode 100644 index 000000000..810456200 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0/report.json new file mode 100644 index 000000000..232caab6a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:16.936182", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..704d7d218 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:16.936182", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.447468" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..204ac32e0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:15.700493", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023475" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..3ebdb92a5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..231e00372 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:15.700493", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9fcc2a4bd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:15.700493", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.433892" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..fcebf13d7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:15.962687", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023430" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0.toml new file mode 100644 index 000000000..a1f4daf9d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0/report.json new file mode 100644 index 000000000..65b7999ae --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:15.962687", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..46b9d1855 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:15.962687", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.435584" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8ba411d31 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:17.779232", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023298" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..b422ea477 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..c0ce5f682 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:17.779232", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f81343fae --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:17.779232", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.465491" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ba9b7c54f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:16.838062", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023005" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0.toml new file mode 100644 index 000000000..9ae3fb7b7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0/report.json new file mode 100644 index 000000000..ed9cd7b72 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:16.838062", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..463950df0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:16.838062", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.421543" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3b2092ff4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5112224822345067, + "mae": 0.40589213371276855, + "r2": 0.5101465582847595, + "score": -0.5112224822345067 + }, + "val": { + "rmse": 0.5583881453279098, + "mae": 0.4439501166343689, + "r2": 0.412036657333374, + "score": -0.5583881453279098 + }, + "test": { + "rmse": 0.5557654209580059, + "mae": 0.4505227506160736, + "r2": 0.42443811893463135, + "score": -0.5557654209580059 + } + }, + "time": "0:07:17.037029", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.024109" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..dad6cd011 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..243854e1b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-evaluation/0/report.json @@ -0,0 +1,171 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5040057771555991, + "mae": 0.39488768577575684, + "r2": 0.5238790512084961, + "score": -0.5040057771555991 + }, + "val": { + "rmse": 0.5558749906446844, + "mae": 0.4398687481880188, + "r2": 0.4173172116279602, + "score": -0.5558749906446844 + }, + "test": { + "rmse": 0.5496162918403876, + "mae": 0.44109272956848145, + "r2": 0.4371039867401123, + "score": -0.5496162918403876 + } + }, + "time": "0:07:17.037029", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b1095bbb7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r1_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 6761248, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 6980 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5042286809765437, + "mae": 0.39605751633644104, + "r2": 0.5234577655792236, + "score": -0.5042286809765437 + }, + "val": { + "rmse": 0.5553348954063525, + "mae": 0.44016024470329285, + "r2": 0.4184489846229553, + "score": -0.5553348954063525 + }, + "test": { + "rmse": 0.550360417400049, + "mae": 0.44305530190467834, + "r2": 0.43557870388031006, + "score": -0.550360417400049 + } + }, + "time": "0:07:17.037029", + "diversity": { + "train": { + "mean_centered_corr": -0.020328260958194733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06901570409536362 + }, + "val": { + "mean_centered_corr": -0.020442906767129898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07035460323095322 + }, + "test": { + "mean_centered_corr": -0.020237648859620094, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06958991289138794 + } + }, + "chunk_size": null, + "eval_batch_size": 16384, + "heads": [ + 13, + 6, + 10, + 14, + 26, + 27, + 18, + 25, + 6, + 15, + 8, + 17, + 10, + 26 + ], + "head_selection_time": "0:00:00.446950" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..17f4e1dc6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5159634144292107, + "mae": 0.410226047039032, + "r2": 0.5010189414024353, + "score": -0.5159634144292107 + }, + "val": { + "rmse": 0.5593939165294649, + "mae": 0.4475388526916504, + "r2": 0.40991663932800293, + "score": -0.5593939165294649 + }, + "test": { + "rmse": 0.5608508784321002, + "mae": 0.45661959052085876, + "r2": 0.41385674476623535, + "score": -0.5608508784321002 + } + }, + "time": "0:10:51.385954", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.023221" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0.toml new file mode 100644 index 000000000..71616aded --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0/report.json new file mode 100644 index 000000000..a0394b28b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5088939009594995, + "mae": 0.40171560645103455, + "r2": 0.5145988464355469, + "score": -0.5088939009594995 + }, + "val": { + "rmse": 0.5558022325623276, + "mae": 0.44188886880874634, + "r2": 0.4174697995185852, + "score": -0.5558022325623276 + }, + "test": { + "rmse": 0.5512480144508224, + "mae": 0.4457772672176361, + "r2": 0.4337567090988159, + "score": -0.5512480144508224 + } + }, + "time": "0:10:51.385954", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1e2adcd92 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5086880691661995, + "mae": 0.4017886221408844, + "r2": 0.5149915218353271, + "score": -0.5086880691661995 + }, + "val": { + "rmse": 0.5554104781133956, + "mae": 0.44194847345352173, + "r2": 0.4182906746864319, + "score": -0.5554104781133956 + }, + "test": { + "rmse": 0.5525622419967886, + "mae": 0.4474184215068817, + "r2": 0.4310535788536072, + "score": -0.5525622419967886 + } + }, + "time": "0:10:51.385954", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4, + 13, + 18, + 11, + 15, + 6, + 21, + 19, + 29, + 31, + 16, + 14, + 22 + ], + "head_selection_time": "0:00:00.387723" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..88166125f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5159634144292107, + "mae": 0.410226047039032, + "r2": 0.5010189414024353, + "score": -0.5159634144292107 + }, + "val": { + "rmse": 0.5593939165294649, + "mae": 0.4475388526916504, + "r2": 0.40991663932800293, + "score": -0.5593939165294649 + }, + "test": { + "rmse": 0.5608508784321002, + "mae": 0.45661959052085876, + "r2": 0.41385674476623535, + "score": -0.5608508784321002 + } + }, + "time": "0:10:52.470388", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.023417" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..fd11e9380 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..aa06b0f20 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5088939009594995, + "mae": 0.40171560645103455, + "r2": 0.5145988464355469, + "score": -0.5088939009594995 + }, + "val": { + "rmse": 0.5558022325623276, + "mae": 0.44188886880874634, + "r2": 0.4174697995185852, + "score": -0.5558022325623276 + }, + "test": { + "rmse": 0.5512480144508224, + "mae": 0.4457772672176361, + "r2": 0.4337567090988159, + "score": -0.5512480144508224 + } + }, + "time": "0:10:52.470388", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..41ea18a38 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5086880691661995, + "mae": 0.4017886221408844, + "r2": 0.5149915218353271, + "score": -0.5086880691661995 + }, + "val": { + "rmse": 0.5554104781133956, + "mae": 0.44194847345352173, + "r2": 0.4182906746864319, + "score": -0.5554104781133956 + }, + "test": { + "rmse": 0.5525622419967886, + "mae": 0.4474184215068817, + "r2": 0.4310535788536072, + "score": -0.5525622419967886 + } + }, + "time": "0:10:52.470388", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4, + 13, + 18, + 11, + 15, + 6, + 21, + 19, + 29, + 31, + 16, + 14, + 22 + ], + "head_selection_time": "0:00:00.393613" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..334f39a2a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.502955286059696, + "mae": 0.3946622908115387, + "r2": 0.5258617401123047, + "score": -0.502955286059696 + }, + "val": { + "rmse": 0.5596610856119687, + "mae": 0.44277745485305786, + "r2": 0.4093528389930725, + "score": -0.5596610856119687 + }, + "test": { + "rmse": 0.5563749515907954, + "mae": 0.44843393564224243, + "r2": 0.4231749176979065, + "score": -0.5563749515907954 + } + }, + "time": "0:12:43.917677", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023354" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0.toml new file mode 100644 index 000000000..6079ea513 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0/report.json new file mode 100644 index 000000000..9550fb3d3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4956110911684622, + "mae": 0.3902001976966858, + "r2": 0.5396074056625366, + "score": -0.4956110911684622 + }, + "val": { + "rmse": 0.5558360123700841, + "mae": 0.43872880935668945, + "r2": 0.41739898920059204, + "score": -0.5558360123700841 + }, + "test": { + "rmse": 0.5491424149361828, + "mae": 0.4410218894481659, + "r2": 0.43807417154312134, + "score": -0.5491424149361828 + } + }, + "time": "0:12:43.917677", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..77d313c47 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4960975196274642, + "mae": 0.3916766345500946, + "r2": 0.5387032628059387, + "score": -0.4960975196274642 + }, + "val": { + "rmse": 0.5552555453020565, + "mae": 0.4401964545249939, + "r2": 0.4186151623725891, + "score": -0.5552555453020565 + }, + "test": { + "rmse": 0.551575730248866, + "mae": 0.44500499963760376, + "r2": 0.43308329582214355, + "score": -0.551575730248866 + } + }, + "time": "0:12:43.917677", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 10, + 2, + 20, + 16, + 6, + 26, + 17, + 21, + 15, + 1, + 10, + 22, + 2, + 30, + 31 + ], + "head_selection_time": "0:00:00.511154" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..249e8ed6e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.502955286059696, + "mae": 0.3946622908115387, + "r2": 0.5258617401123047, + "score": -0.502955286059696 + }, + "val": { + "rmse": 0.5596610856119687, + "mae": 0.44277745485305786, + "r2": 0.4093528389930725, + "score": -0.5596610856119687 + }, + "test": { + "rmse": 0.5563749515907954, + "mae": 0.44843393564224243, + "r2": 0.4231749176979065, + "score": -0.5563749515907954 + } + }, + "time": "0:12:43.404674", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023207" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..d0652b358 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..b98eef892 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4956110911684622, + "mae": 0.3902001976966858, + "r2": 0.5396074056625366, + "score": -0.4956110911684622 + }, + "val": { + "rmse": 0.5558360123700841, + "mae": 0.43872880935668945, + "r2": 0.41739898920059204, + "score": -0.5558360123700841 + }, + "test": { + "rmse": 0.5491424149361828, + "mae": 0.4410218894481659, + "r2": 0.43807417154312134, + "score": -0.5491424149361828 + } + }, + "time": "0:12:43.404674", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8d7085876 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4960975196274642, + "mae": 0.3916766345500946, + "r2": 0.5387032628059387, + "score": -0.4960975196274642 + }, + "val": { + "rmse": 0.5552555453020565, + "mae": 0.4401964545249939, + "r2": 0.4186151623725891, + "score": -0.5552555453020565 + }, + "test": { + "rmse": 0.551575730248866, + "mae": 0.44500499963760376, + "r2": 0.43308329582214355, + "score": -0.551575730248866 + } + }, + "time": "0:12:43.404674", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 10, + 2, + 20, + 16, + 6, + 26, + 17, + 21, + 15, + 1, + 10, + 22, + 2, + 30, + 31 + ], + "head_selection_time": "0:00:00.511173" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1ea412fe3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5127050489681885, + "mae": 0.40456077456474304, + "r2": 0.5073012113571167, + "score": -0.5127050489681885 + }, + "val": { + "rmse": 0.5586283212914263, + "mae": 0.4427802264690399, + "r2": 0.41153067350387573, + "score": -0.5586283212914263 + }, + "test": { + "rmse": 0.5508851568810451, + "mae": 0.4434031546115875, + "r2": 0.4345019459724426, + "score": -0.5508851568810451 + } + }, + "time": "0:10:54.308917", + "diversity": { + "train": { + "mean_centered_corr": -0.020132694393396378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07372817397117615 + }, + "val": { + "mean_centered_corr": -0.02042890153825283, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07545945793390274 + }, + "test": { + "mean_centered_corr": -0.020171180367469788, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07420117408037186 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 10 + ], + "head_selection_time": "0:00:00.023488" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0.toml new file mode 100644 index 000000000..f7b19054f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0/report.json new file mode 100644 index 000000000..151ef7a99 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5087820041628177, + "mae": 0.4012722671031952, + "r2": 0.5148123502731323, + "score": -0.5087820041628177 + }, + "val": { + "rmse": 0.5555827210327928, + "mae": 0.44155657291412354, + "r2": 0.41792982816696167, + "score": -0.5555827210327928 + }, + "test": { + "rmse": 0.5515477953887775, + "mae": 0.44571202993392944, + "r2": 0.43314069509506226, + "score": -0.5515477953887775 + } + }, + "time": "0:10:54.308917", + "diversity": { + "train": { + "mean_centered_corr": -0.020132694393396378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07372817397117615 + }, + "val": { + "mean_centered_corr": -0.02042890153825283, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07545945793390274 + }, + "test": { + "mean_centered_corr": -0.020171180367469788, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07420117408037186 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..af1922030 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,188 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5091947669637598, + "mae": 0.40187934041023254, + "r2": 0.5140247344970703, + "score": -0.5091947669637598 + }, + "val": { + "rmse": 0.5550703688421238, + "mae": 0.44088995456695557, + "r2": 0.41900283098220825, + "score": -0.5550703688421238 + }, + "test": { + "rmse": 0.5524078860674355, + "mae": 0.44680875539779663, + "r2": 0.4313713312149048, + "score": -0.5524078860674355 + } + }, + "time": "0:10:54.308917", + "diversity": { + "train": { + "mean_centered_corr": -0.020132694393396378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07372817397117615 + }, + "val": { + "mean_centered_corr": -0.02042890153825283, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07545945793390274 + }, + "test": { + "mean_centered_corr": -0.020171180367469788, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07420117408037186 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 10, + 14, + 15, + 25, + 6, + 18, + 26, + 21, + 13, + 4 + ], + "head_selection_time": "0:00:00.287558" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..102ae55df --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5159634144292107, + "mae": 0.410226047039032, + "r2": 0.5010189414024353, + "score": -0.5159634144292107 + }, + "val": { + "rmse": 0.5593939165294649, + "mae": 0.4475388526916504, + "r2": 0.40991663932800293, + "score": -0.5593939165294649 + }, + "test": { + "rmse": 0.5608508784321002, + "mae": 0.45661959052085876, + "r2": 0.41385674476623535, + "score": -0.5608508784321002 + } + }, + "time": "0:10:53.683873", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.023809" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..3c50600be --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..13022f6f8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5088939009594995, + "mae": 0.40171560645103455, + "r2": 0.5145988464355469, + "score": -0.5088939009594995 + }, + "val": { + "rmse": 0.5558022325623276, + "mae": 0.44188886880874634, + "r2": 0.4174697995185852, + "score": -0.5558022325623276 + }, + "test": { + "rmse": 0.5512480144508224, + "mae": 0.4457772672176361, + "r2": 0.4337567090988159, + "score": -0.5512480144508224 + } + }, + "time": "0:10:53.683873", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..42bfb1dbd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5086880691661995, + "mae": 0.4017886221408844, + "r2": 0.5149915218353271, + "score": -0.5086880691661995 + }, + "val": { + "rmse": 0.5554104781133956, + "mae": 0.44194847345352173, + "r2": 0.4182906746864319, + "score": -0.5554104781133956 + }, + "test": { + "rmse": 0.5525622419967886, + "mae": 0.4474184215068817, + "r2": 0.4310535788536072, + "score": -0.5525622419967886 + } + }, + "time": "0:10:53.683873", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4, + 13, + 18, + 11, + 15, + 6, + 21, + 19, + 29, + 31, + 16, + 14, + 22 + ], + "head_selection_time": "0:00:00.412208" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b80356461 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5123672452974037, + "mae": 0.4070248603820801, + "r2": 0.5079503059387207, + "score": -0.5123672452974037 + }, + "val": { + "rmse": 0.5584356978034605, + "mae": 0.4474104344844818, + "r2": 0.4119364619255066, + "score": -0.5584356978034605 + }, + "test": { + "rmse": 0.5561328378795123, + "mae": 0.4517236351966858, + "r2": 0.42367684841156006, + "score": -0.5561328378795123 + } + }, + "time": "0:10:53.886867", + "diversity": { + "train": { + "mean_centered_corr": -0.020328762009739876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07261504977941513 + }, + "val": { + "mean_centered_corr": -0.020213842391967773, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07467607408761978 + }, + "test": { + "mean_centered_corr": -0.0204370878636837, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07374871522188187 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 10 + ], + "head_selection_time": "0:00:00.023041" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0.toml new file mode 100644 index 000000000..7a4a15b7a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0/report.json new file mode 100644 index 000000000..0f7eca095 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5069226317597442, + "mae": 0.3994384706020355, + "r2": 0.5183521509170532, + "score": -0.5069226317597442 + }, + "val": { + "rmse": 0.5557252820018314, + "mae": 0.44164222478866577, + "r2": 0.4176310896873474, + "score": -0.5557252820018314 + }, + "test": { + "rmse": 0.5515971532207388, + "mae": 0.44512292742729187, + "r2": 0.43303918838500977, + "score": -0.5515971532207388 + } + }, + "time": "0:10:53.886867", + "diversity": { + "train": { + "mean_centered_corr": -0.020328762009739876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07261504977941513 + }, + "val": { + "mean_centered_corr": -0.020213842391967773, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07467607408761978 + }, + "test": { + "mean_centered_corr": -0.0204370878636837, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07374871522188187 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..dd358b3a9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,185 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5084577129451613, + "mae": 0.4028453230857849, + "r2": 0.5154306292533875, + "score": -0.5084577129451613 + }, + "val": { + "rmse": 0.555272076390732, + "mae": 0.44189491868019104, + "r2": 0.4185805320739746, + "score": -0.555272076390732 + }, + "test": { + "rmse": 0.5536843483543759, + "mae": 0.4483037292957306, + "r2": 0.4287404417991638, + "score": -0.5536843483543759 + } + }, + "time": "0:10:53.886867", + "diversity": { + "train": { + "mean_centered_corr": -0.020328762009739876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07261504977941513 + }, + "val": { + "mean_centered_corr": -0.020213842391967773, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07467607408761978 + }, + "test": { + "mean_centered_corr": -0.0204370878636837, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07374871522188187 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 10, + 6, + 4, + 13, + 8, + 25, + 18 + ], + "head_selection_time": "0:00:00.203331" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f6a3eac71 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.502955286059696, + "mae": 0.3946622908115387, + "r2": 0.5258617401123047, + "score": -0.502955286059696 + }, + "val": { + "rmse": 0.5596610856119687, + "mae": 0.44277745485305786, + "r2": 0.4093528389930725, + "score": -0.5596610856119687 + }, + "test": { + "rmse": 0.5563749515907954, + "mae": 0.44843393564224243, + "r2": 0.4231749176979065, + "score": -0.5563749515907954 + } + }, + "time": "0:12:42.623804", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023268" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..e79913e2b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..f3b4f49f6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4956110911684622, + "mae": 0.3902001976966858, + "r2": 0.5396074056625366, + "score": -0.4956110911684622 + }, + "val": { + "rmse": 0.5558360123700841, + "mae": 0.43872880935668945, + "r2": 0.41739898920059204, + "score": -0.5558360123700841 + }, + "test": { + "rmse": 0.5491424149361828, + "mae": 0.4410218894481659, + "r2": 0.43807417154312134, + "score": -0.5491424149361828 + } + }, + "time": "0:12:42.623804", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7de17cfbc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4960975196274642, + "mae": 0.3916766345500946, + "r2": 0.5387032628059387, + "score": -0.4960975196274642 + }, + "val": { + "rmse": 0.5552555453020565, + "mae": 0.4401964545249939, + "r2": 0.4186151623725891, + "score": -0.5552555453020565 + }, + "test": { + "rmse": 0.551575730248866, + "mae": 0.44500499963760376, + "r2": 0.43308329582214355, + "score": -0.551575730248866 + } + }, + "time": "0:12:42.623804", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 10, + 2, + 20, + 16, + 6, + 26, + 17, + 21, + 15, + 1, + 10, + 22, + 2, + 30, + 31 + ], + "head_selection_time": "0:00:00.507256" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f552fa0c2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.5090104864160705, + "mae": 0.4024014472961426, + "r2": 0.5143764615058899, + "score": -0.5090104864160705 + }, + "val": { + "rmse": 0.5603404179951016, + "mae": 0.4449836015701294, + "r2": 0.4079180955886841, + "score": -0.5603404179951016 + }, + "test": { + "rmse": 0.5552489703004402, + "mae": 0.4475087821483612, + "r2": 0.4255073070526123, + "score": -0.5552489703004402 + } + }, + "time": "0:12:41.926992", + "diversity": { + "train": { + "mean_centered_corr": -0.020042935386300087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0816296711564064 + }, + "val": { + "mean_centered_corr": -0.02009006403386593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0869147852063179 + }, + "test": { + "mean_centered_corr": -0.020102225244045258, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08457230031490326 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 1 + ], + "head_selection_time": "0:00:00.023517" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0.toml new file mode 100644 index 000000000..7d30e317b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0/report.json new file mode 100644 index 000000000..fd1873d88 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.49865310275031377, + "mae": 0.3922213613986969, + "r2": 0.5339384078979492, + "score": -0.49865310275031377 + }, + "val": { + "rmse": 0.5560690371152802, + "mae": 0.438926637172699, + "r2": 0.4169103503227234, + "score": -0.5560690371152802 + }, + "test": { + "rmse": 0.5483022050176959, + "mae": 0.4402907192707062, + "r2": 0.4397923946380615, + "score": -0.5483022050176959 + } + }, + "time": "0:12:41.926992", + "diversity": { + "train": { + "mean_centered_corr": -0.020042935386300087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0816296711564064 + }, + "val": { + "mean_centered_corr": -0.02009006403386593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0869147852063179 + }, + "test": { + "mean_centered_corr": -0.020102225244045258, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08457230031490326 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c50e32901 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,190 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.5000589753179171, + "mae": 0.3948548436164856, + "r2": 0.5313067436218262, + "score": -0.5000589753179171 + }, + "val": { + "rmse": 0.5554431013363782, + "mae": 0.4404180943965912, + "r2": 0.4182223081588745, + "score": -0.5554431013363782 + }, + "test": { + "rmse": 0.5505825252366326, + "mae": 0.44418030977249146, + "r2": 0.435123085975647, + "score": -0.5505825252366326 + } + }, + "time": "0:12:41.926992", + "diversity": { + "train": { + "mean_centered_corr": -0.020042935386300087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0816296711564064 + }, + "val": { + "mean_centered_corr": -0.02009006403386593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0869147852063179 + }, + "test": { + "mean_centered_corr": -0.020102225244045258, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08457230031490326 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 1, + 13, + 22, + 18, + 17, + 20, + 11, + 12, + 29, + 6, + 14, + 26 + ], + "head_selection_time": "0:00:00.374219" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..96a17f586 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5159634144292107, + "mae": 0.410226047039032, + "r2": 0.5010189414024353, + "score": -0.5159634144292107 + }, + "val": { + "rmse": 0.5593939165294649, + "mae": 0.4475388526916504, + "r2": 0.40991663932800293, + "score": -0.5593939165294649 + }, + "test": { + "rmse": 0.5608508784321002, + "mae": 0.45661959052085876, + "r2": 0.41385674476623535, + "score": -0.5608508784321002 + } + }, + "time": "0:10:54.207858", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.023310" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..210a8f378 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..1fee1c9c3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5088939009594995, + "mae": 0.40171560645103455, + "r2": 0.5145988464355469, + "score": -0.5088939009594995 + }, + "val": { + "rmse": 0.5558022325623276, + "mae": 0.44188886880874634, + "r2": 0.4174697995185852, + "score": -0.5558022325623276 + }, + "test": { + "rmse": 0.5512480144508224, + "mae": 0.4457772672176361, + "r2": 0.4337567090988159, + "score": -0.5512480144508224 + } + }, + "time": "0:10:54.207858", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5be345413 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5086880691661995, + "mae": 0.4017886221408844, + "r2": 0.5149915218353271, + "score": -0.5086880691661995 + }, + "val": { + "rmse": 0.5554104781133956, + "mae": 0.44194847345352173, + "r2": 0.4182906746864319, + "score": -0.5554104781133956 + }, + "test": { + "rmse": 0.5525622419967886, + "mae": 0.4474184215068817, + "r2": 0.4310535788536072, + "score": -0.5525622419967886 + } + }, + "time": "0:10:54.207858", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4, + 13, + 18, + 11, + 15, + 6, + 21, + 19, + 29, + 31, + 16, + 14, + 22 + ], + "head_selection_time": "0:00:00.393665" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..368c5c71a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5119769486010273, + "mae": 0.40593889355659485, + "r2": 0.5086996555328369, + "score": -0.5119769486010273 + }, + "val": { + "rmse": 0.5583888658501774, + "mae": 0.4450767934322357, + "r2": 0.41203510761260986, + "score": -0.5583888658501774 + }, + "test": { + "rmse": 0.5543243670215346, + "mae": 0.44926270842552185, + "r2": 0.4274190068244934, + "score": -0.5543243670215346 + } + }, + "time": "0:10:54.579717", + "diversity": { + "train": { + "mean_centered_corr": -0.02000253275036812, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07106093317270279 + }, + "val": { + "mean_centered_corr": -0.019867246970534325, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07228023558855057 + }, + "test": { + "mean_centered_corr": -0.020095540210604668, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07158966362476349 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 10 + ], + "head_selection_time": "0:00:00.023234" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0.toml new file mode 100644 index 000000000..cc9c6b92e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0/report.json new file mode 100644 index 000000000..83fb0c1c4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5068120050384366, + "mae": 0.3983139991760254, + "r2": 0.5185623168945312, + "score": -0.5068120050384366 + }, + "val": { + "rmse": 0.5557362219646558, + "mae": 0.4406058192253113, + "r2": 0.4176081418991089, + "score": -0.5557362219646558 + }, + "test": { + "rmse": 0.5505742164168008, + "mae": 0.4435580372810364, + "r2": 0.43514013290405273, + "score": -0.5505742164168008 + } + }, + "time": "0:10:54.579717", + "diversity": { + "train": { + "mean_centered_corr": -0.02000253275036812, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07106093317270279 + }, + "val": { + "mean_centered_corr": -0.019867246970534325, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07228023558855057 + }, + "test": { + "mean_centered_corr": -0.020095540210604668, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07158966362476349 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..dc718bf4b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5072879723238778, + "mae": 0.39973556995391846, + "r2": 0.5176576375961304, + "score": -0.5072879723238778 + }, + "val": { + "rmse": 0.5551940861270195, + "mae": 0.44103240966796875, + "r2": 0.4187438488006592, + "score": -0.5551940861270195 + }, + "test": { + "rmse": 0.5519268755870242, + "mae": 0.44590064883232117, + "r2": 0.4323612451553345, + "score": -0.5519268755870242 + } + }, + "time": "0:10:54.579717", + "diversity": { + "train": { + "mean_centered_corr": -0.02000253275036812, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07106093317270279 + }, + "val": { + "mean_centered_corr": -0.019867246970534325, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07228023558855057 + }, + "test": { + "mean_centered_corr": -0.020095540210604668, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07158966362476349 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 10, + 13, + 21, + 26, + 25, + 18, + 14, + 6, + 11, + 29, + 13, + 4, + 19, + 21, + 16, + 26 + ], + "head_selection_time": "0:00:00.520164" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3de887292 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.502955286059696, + "mae": 0.3946622908115387, + "r2": 0.5258617401123047, + "score": -0.502955286059696 + }, + "val": { + "rmse": 0.5596610856119687, + "mae": 0.44277745485305786, + "r2": 0.4093528389930725, + "score": -0.5596610856119687 + }, + "test": { + "rmse": 0.5563749515907954, + "mae": 0.44843393564224243, + "r2": 0.4231749176979065, + "score": -0.5563749515907954 + } + }, + "time": "0:12:40.834182", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023278" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..ad04ceb1e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..0fb4e76c5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4956110911684622, + "mae": 0.3902001976966858, + "r2": 0.5396074056625366, + "score": -0.4956110911684622 + }, + "val": { + "rmse": 0.5558360123700841, + "mae": 0.43872880935668945, + "r2": 0.41739898920059204, + "score": -0.5558360123700841 + }, + "test": { + "rmse": 0.5491424149361828, + "mae": 0.4410218894481659, + "r2": 0.43807417154312134, + "score": -0.5491424149361828 + } + }, + "time": "0:12:40.834182", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..343dbebb6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r2_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7056800, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4960975196274642, + "mae": 0.3916766345500946, + "r2": 0.5387032628059387, + "score": -0.4960975196274642 + }, + "val": { + "rmse": 0.5552555453020565, + "mae": 0.4401964545249939, + "r2": 0.4186151623725891, + "score": -0.5552555453020565 + }, + "test": { + "rmse": 0.551575730248866, + "mae": 0.44500499963760376, + "r2": 0.43308329582214355, + "score": -0.551575730248866 + } + }, + "time": "0:12:40.834182", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 10, + 2, + 20, + 16, + 6, + 26, + 17, + 21, + 15, + 1, + 10, + 22, + 2, + 30, + 31 + ], + "head_selection_time": "0:00:00.523278" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..d33af28cf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5159634144292107, + "mae": 0.410226047039032, + "r2": 0.5010189414024353, + "score": -0.5159634144292107 + }, + "val": { + "rmse": 0.5593939165294649, + "mae": 0.4475388526916504, + "r2": 0.40991663932800293, + "score": -0.5593939165294649 + }, + "test": { + "rmse": 0.5608508784321002, + "mae": 0.45661959052085876, + "r2": 0.41385674476623535, + "score": -0.5608508784321002 + } + }, + "time": "0:16:51.845095", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.023303" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0.toml new file mode 100644 index 000000000..815cb9308 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0/report.json new file mode 100644 index 000000000..f3e58291a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5088939009594995, + "mae": 0.40171560645103455, + "r2": 0.5145988464355469, + "score": -0.5088939009594995 + }, + "val": { + "rmse": 0.5558022325623276, + "mae": 0.44188886880874634, + "r2": 0.4174697995185852, + "score": -0.5558022325623276 + }, + "test": { + "rmse": 0.5512480144508224, + "mae": 0.4457772672176361, + "r2": 0.4337567090988159, + "score": -0.5512480144508224 + } + }, + "time": "0:16:51.845095", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3b7b383d1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5086880691661995, + "mae": 0.4017886221408844, + "r2": 0.5149915218353271, + "score": -0.5086880691661995 + }, + "val": { + "rmse": 0.5554104781133956, + "mae": 0.44194847345352173, + "r2": 0.4182906746864319, + "score": -0.5554104781133956 + }, + "test": { + "rmse": 0.5525622419967886, + "mae": 0.4474184215068817, + "r2": 0.4310535788536072, + "score": -0.5525622419967886 + } + }, + "time": "0:16:51.845095", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4, + 13, + 18, + 11, + 15, + 6, + 21, + 19, + 29, + 31, + 16, + 14, + 22 + ], + "head_selection_time": "0:00:00.393834" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4d4007f7b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5159634144292107, + "mae": 0.410226047039032, + "r2": 0.5010189414024353, + "score": -0.5159634144292107 + }, + "val": { + "rmse": 0.5593939165294649, + "mae": 0.4475388526916504, + "r2": 0.40991663932800293, + "score": -0.5593939165294649 + }, + "test": { + "rmse": 0.5608508784321002, + "mae": 0.45661959052085876, + "r2": 0.41385674476623535, + "score": -0.5608508784321002 + } + }, + "time": "0:16:52.052732", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.023593" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..88ef7e1b1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..cdb6dbee0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5088939009594995, + "mae": 0.40171560645103455, + "r2": 0.5145988464355469, + "score": -0.5088939009594995 + }, + "val": { + "rmse": 0.5558022325623276, + "mae": 0.44188886880874634, + "r2": 0.4174697995185852, + "score": -0.5558022325623276 + }, + "test": { + "rmse": 0.5512480144508224, + "mae": 0.4457772672176361, + "r2": 0.4337567090988159, + "score": -0.5512480144508224 + } + }, + "time": "0:16:52.052732", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2913e4690 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5086880691661995, + "mae": 0.4017886221408844, + "r2": 0.5149915218353271, + "score": -0.5086880691661995 + }, + "val": { + "rmse": 0.5554104781133956, + "mae": 0.44194847345352173, + "r2": 0.4182906746864319, + "score": -0.5554104781133956 + }, + "test": { + "rmse": 0.5525622419967886, + "mae": 0.4474184215068817, + "r2": 0.4310535788536072, + "score": -0.5525622419967886 + } + }, + "time": "0:16:52.052732", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4, + 13, + 18, + 11, + 15, + 6, + 21, + 19, + 29, + 31, + 16, + 14, + 22 + ], + "head_selection_time": "0:00:00.408183" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ff9ee8395 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.502955286059696, + "mae": 0.3946622908115387, + "r2": 0.5258617401123047, + "score": -0.502955286059696 + }, + "val": { + "rmse": 0.5596610856119687, + "mae": 0.44277745485305786, + "r2": 0.4093528389930725, + "score": -0.5596610856119687 + }, + "test": { + "rmse": 0.5563749515907954, + "mae": 0.44843393564224243, + "r2": 0.4231749176979065, + "score": -0.5563749515907954 + } + }, + "time": "0:19:39.258488", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023852" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0.toml new file mode 100644 index 000000000..6025ce67d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0/report.json new file mode 100644 index 000000000..b30eebcd1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4956110911684622, + "mae": 0.3902001976966858, + "r2": 0.5396074056625366, + "score": -0.4956110911684622 + }, + "val": { + "rmse": 0.5558360123700841, + "mae": 0.43872880935668945, + "r2": 0.41739898920059204, + "score": -0.5558360123700841 + }, + "test": { + "rmse": 0.5491424149361828, + "mae": 0.4410218894481659, + "r2": 0.43807417154312134, + "score": -0.5491424149361828 + } + }, + "time": "0:19:39.258488", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f1cf12fa3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4960975196274642, + "mae": 0.3916766345500946, + "r2": 0.5387032628059387, + "score": -0.4960975196274642 + }, + "val": { + "rmse": 0.5552555453020565, + "mae": 0.4401964545249939, + "r2": 0.4186151623725891, + "score": -0.5552555453020565 + }, + "test": { + "rmse": 0.551575730248866, + "mae": 0.44500499963760376, + "r2": 0.43308329582214355, + "score": -0.551575730248866 + } + }, + "time": "0:19:39.258488", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 10, + 2, + 20, + 16, + 6, + 26, + 17, + 21, + 15, + 1, + 10, + 22, + 2, + 30, + 31 + ], + "head_selection_time": "0:00:00.492201" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..cbe0b2a93 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.502955286059696, + "mae": 0.3946622908115387, + "r2": 0.5258617401123047, + "score": -0.502955286059696 + }, + "val": { + "rmse": 0.5596610856119687, + "mae": 0.44277745485305786, + "r2": 0.4093528389930725, + "score": -0.5596610856119687 + }, + "test": { + "rmse": 0.5563749515907954, + "mae": 0.44843393564224243, + "r2": 0.4231749176979065, + "score": -0.5563749515907954 + } + }, + "time": "0:19:40.313994", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023504" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..57515fe2a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..c295956d9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4956110911684622, + "mae": 0.3902001976966858, + "r2": 0.5396074056625366, + "score": -0.4956110911684622 + }, + "val": { + "rmse": 0.5558360123700841, + "mae": 0.43872880935668945, + "r2": 0.41739898920059204, + "score": -0.5558360123700841 + }, + "test": { + "rmse": 0.5491424149361828, + "mae": 0.4410218894481659, + "r2": 0.43807417154312134, + "score": -0.5491424149361828 + } + }, + "time": "0:19:40.313994", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..63c75d98f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4960975196274642, + "mae": 0.3916766345500946, + "r2": 0.5387032628059387, + "score": -0.4960975196274642 + }, + "val": { + "rmse": 0.5552555453020565, + "mae": 0.4401964545249939, + "r2": 0.4186151623725891, + "score": -0.5552555453020565 + }, + "test": { + "rmse": 0.551575730248866, + "mae": 0.44500499963760376, + "r2": 0.43308329582214355, + "score": -0.551575730248866 + } + }, + "time": "0:19:40.313994", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 10, + 2, + 20, + 16, + 6, + 26, + 17, + 21, + 15, + 1, + 10, + 22, + 2, + 30, + 31 + ], + "head_selection_time": "0:00:00.510200" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..166c52623 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.5089933603917912, + "mae": 0.4041629731655121, + "r2": 0.5144091844558716, + "score": -0.5089933603917912 + }, + "val": { + "rmse": 0.56004920160056, + "mae": 0.44279050827026367, + "r2": 0.40853333473205566, + "score": -0.56004920160056 + }, + "test": { + "rmse": 0.5537449525211285, + "mae": 0.4460917115211487, + "r2": 0.4286153316497803, + "score": -0.5537449525211285 + } + }, + "time": "0:19:35.794115", + "diversity": { + "train": { + "mean_centered_corr": -0.019954748451709747, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08140051364898682 + }, + "val": { + "mean_centered_corr": -0.020047752186655998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0873202458024025 + }, + "test": { + "mean_centered_corr": -0.019868871197104454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08496350795030594 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 18 + ], + "head_selection_time": "0:00:00.024205" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0.toml new file mode 100644 index 000000000..5c7b4aa29 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0/report.json new file mode 100644 index 000000000..fbc6695c7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4982967871668599, + "mae": 0.3913329243659973, + "r2": 0.5346041917800903, + "score": -0.4982967871668599 + }, + "val": { + "rmse": 0.5564917917545931, + "mae": 0.4382628798484802, + "r2": 0.4160234332084656, + "score": -0.5564917917545931 + }, + "test": { + "rmse": 0.5481810097693468, + "mae": 0.4391641318798065, + "r2": 0.44004005193710327, + "score": -0.5481810097693468 + } + }, + "time": "0:19:35.794115", + "diversity": { + "train": { + "mean_centered_corr": -0.019954748451709747, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08140051364898682 + }, + "val": { + "mean_centered_corr": -0.020047752186655998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0873202458024025 + }, + "test": { + "mean_centered_corr": -0.019868871197104454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08496350795030594 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..15592bdf7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,189 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4995550321442935, + "mae": 0.3944211006164551, + "r2": 0.5322508811950684, + "score": -0.4995550321442935 + }, + "val": { + "rmse": 0.5553600907849848, + "mae": 0.4404390752315521, + "r2": 0.4183961749076843, + "score": -0.5553600907849848 + }, + "test": { + "rmse": 0.5504577714884789, + "mae": 0.44397687911987305, + "r2": 0.4353790283203125, + "score": -0.5504577714884789 + } + }, + "time": "0:19:35.794115", + "diversity": { + "train": { + "mean_centered_corr": -0.019954748451709747, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08140051364898682 + }, + "val": { + "mean_centered_corr": -0.020047752186655998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0873202458024025 + }, + "test": { + "mean_centered_corr": -0.019868871197104454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08496350795030594 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 18, + 1, + 12, + 5, + 25, + 13, + 14, + 15, + 6, + 0, + 11 + ], + "head_selection_time": "0:00:00.337718" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ea89cd2b5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5159634144292107, + "mae": 0.410226047039032, + "r2": 0.5010189414024353, + "score": -0.5159634144292107 + }, + "val": { + "rmse": 0.5593939165294649, + "mae": 0.4475388526916504, + "r2": 0.40991663932800293, + "score": -0.5593939165294649 + }, + "test": { + "rmse": 0.5608508784321002, + "mae": 0.45661959052085876, + "r2": 0.41385674476623535, + "score": -0.5608508784321002 + } + }, + "time": "0:16:49.735827", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.023311" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..3cf5b599b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..b587ef7c9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5088939009594995, + "mae": 0.40171560645103455, + "r2": 0.5145988464355469, + "score": -0.5088939009594995 + }, + "val": { + "rmse": 0.5558022325623276, + "mae": 0.44188886880874634, + "r2": 0.4174697995185852, + "score": -0.5558022325623276 + }, + "test": { + "rmse": 0.5512480144508224, + "mae": 0.4457772672176361, + "r2": 0.4337567090988159, + "score": -0.5512480144508224 + } + }, + "time": "0:16:49.735827", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..83873e8d4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5086880691661995, + "mae": 0.4017886221408844, + "r2": 0.5149915218353271, + "score": -0.5086880691661995 + }, + "val": { + "rmse": 0.5554104781133956, + "mae": 0.44194847345352173, + "r2": 0.4182906746864319, + "score": -0.5554104781133956 + }, + "test": { + "rmse": 0.5525622419967886, + "mae": 0.4474184215068817, + "r2": 0.4310535788536072, + "score": -0.5525622419967886 + } + }, + "time": "0:16:49.735827", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4, + 13, + 18, + 11, + 15, + 6, + 21, + 19, + 29, + 31, + 16, + 14, + 22 + ], + "head_selection_time": "0:00:00.404210" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e5b025742 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5175918577279733, + "mae": 0.40775081515312195, + "r2": 0.4978642463684082, + "score": -0.5175918577279733 + }, + "val": { + "rmse": 0.5572652691682626, + "mae": 0.44076380133628845, + "r2": 0.4143989086151123, + "score": -0.5572652691682626 + }, + "test": { + "rmse": 0.5524207799310179, + "mae": 0.4452172815799713, + "r2": 0.4313448667526245, + "score": -0.5524207799310179 + } + }, + "time": "0:14:43.657532", + "diversity": { + "train": { + "mean_centered_corr": -0.02016017585992813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06105273962020874 + }, + "val": { + "mean_centered_corr": -0.01993357017636299, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06221357360482216 + }, + "test": { + "mean_centered_corr": -0.020221160724759102, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06099524348974228 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 11 + ], + "head_selection_time": "0:00:00.023842" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0.toml new file mode 100644 index 000000000..f15a53b07 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0/report.json new file mode 100644 index 000000000..ddf9415fb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5150130136938161, + "mae": 0.40628108382225037, + "r2": 0.5028554797172546, + "score": -0.5150130136938161 + }, + "val": { + "rmse": 0.5559356776481553, + "mae": 0.441616415977478, + "r2": 0.4171900153160095, + "score": -0.5559356776481553 + }, + "test": { + "rmse": 0.55135486033622, + "mae": 0.445721298456192, + "r2": 0.43353718519210815, + "score": -0.55135486033622 + } + }, + "time": "0:14:43.657532", + "diversity": { + "train": { + "mean_centered_corr": -0.02016017585992813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06105273962020874 + }, + "val": { + "mean_centered_corr": -0.01993357017636299, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06221357360482216 + }, + "test": { + "mean_centered_corr": -0.020221160724759102, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06099524348974228 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f95c28851 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,187 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5148489632002486, + "mae": 0.40648993849754333, + "r2": 0.5031721591949463, + "score": -0.5148489632002486 + }, + "val": { + "rmse": 0.5553930389270275, + "mae": 0.4419495463371277, + "r2": 0.4183272123336792, + "score": -0.5553930389270275 + }, + "test": { + "rmse": 0.5519483119271499, + "mae": 0.4472074508666992, + "r2": 0.4323171377182007, + "score": -0.5519483119271499 + } + }, + "time": "0:14:43.657532", + "diversity": { + "train": { + "mean_centered_corr": -0.02016017585992813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06105273962020874 + }, + "val": { + "mean_centered_corr": -0.01993357017636299, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06221357360482216 + }, + "test": { + "mean_centered_corr": -0.020221160724759102, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06099524348974228 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 11, + 10, + 26, + 21, + 15, + 16, + 31, + 19, + 13 + ], + "head_selection_time": "0:00:00.256647" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..48e0ae5f6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.502955286059696, + "mae": 0.3946622908115387, + "r2": 0.5258617401123047, + "score": -0.502955286059696 + }, + "val": { + "rmse": 0.5596610856119687, + "mae": 0.44277745485305786, + "r2": 0.4093528389930725, + "score": -0.5596610856119687 + }, + "test": { + "rmse": 0.5563749515907954, + "mae": 0.44843393564224243, + "r2": 0.4231749176979065, + "score": -0.5563749515907954 + } + }, + "time": "0:19:37.834703", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.024218" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..46d1d4c33 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..1dbd8a906 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4956110911684622, + "mae": 0.3902001976966858, + "r2": 0.5396074056625366, + "score": -0.4956110911684622 + }, + "val": { + "rmse": 0.5558360123700841, + "mae": 0.43872880935668945, + "r2": 0.41739898920059204, + "score": -0.5558360123700841 + }, + "test": { + "rmse": 0.5491424149361828, + "mae": 0.4410218894481659, + "r2": 0.43807417154312134, + "score": -0.5491424149361828 + } + }, + "time": "0:19:37.834703", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0ccda495a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4960975196274642, + "mae": 0.3916766345500946, + "r2": 0.5387032628059387, + "score": -0.4960975196274642 + }, + "val": { + "rmse": 0.5552555453020565, + "mae": 0.4401964545249939, + "r2": 0.4186151623725891, + "score": -0.5552555453020565 + }, + "test": { + "rmse": 0.551575730248866, + "mae": 0.44500499963760376, + "r2": 0.43308329582214355, + "score": -0.551575730248866 + } + }, + "time": "0:19:37.834703", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 10, + 2, + 20, + 16, + 6, + 26, + 17, + 21, + 15, + 1, + 10, + 22, + 2, + 30, + 31 + ], + "head_selection_time": "0:00:00.498576" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..34f9c8081 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.5111900976758429, + "mae": 0.40660035610198975, + "r2": 0.5102086067199707, + "score": -0.5111900976758429 + }, + "val": { + "rmse": 0.5601935253217213, + "mae": 0.4456070363521576, + "r2": 0.40822845697402954, + "score": -0.5601935253217213 + }, + "test": { + "rmse": 0.5598375837203123, + "mae": 0.45322397351264954, + "r2": 0.4159727692604065, + "score": -0.5598375837203123 + } + }, + "time": "0:19:35.624704", + "diversity": { + "train": { + "mean_centered_corr": -0.02013956569135189, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0812845528125763 + }, + "val": { + "mean_centered_corr": -0.02026885561645031, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08767195791006088 + }, + "test": { + "mean_centered_corr": -0.019879044964909554, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08455239981412888 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 1 + ], + "head_selection_time": "0:00:00.023504" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0.toml new file mode 100644 index 000000000..85fb0b1da --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0/report.json new file mode 100644 index 000000000..20fc4d6d9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4985970842566162, + "mae": 0.3925943076610565, + "r2": 0.5340430736541748, + "score": -0.4985970842566162 + }, + "val": { + "rmse": 0.5560380585478252, + "mae": 0.43930554389953613, + "r2": 0.41697531938552856, + "score": -0.5560380585478252 + }, + "test": { + "rmse": 0.5494440227195571, + "mae": 0.44151392579078674, + "r2": 0.43745678663253784, + "score": -0.5494440227195571 + } + }, + "time": "0:19:35.624704", + "diversity": { + "train": { + "mean_centered_corr": -0.02013956569135189, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0812845528125763 + }, + "val": { + "mean_centered_corr": -0.02026885561645031, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08767195791006088 + }, + "test": { + "mean_centered_corr": -0.019879044964909554, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08455239981412888 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0ec7829f3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,190 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4998972608393945, + "mae": 0.39439302682876587, + "r2": 0.5316097736358643, + "score": -0.4998972608393945 + }, + "val": { + "rmse": 0.5553281335127394, + "mae": 0.4403950870037079, + "r2": 0.4184631109237671, + "score": -0.5553281335127394 + }, + "test": { + "rmse": 0.550823697016921, + "mae": 0.44420838356018066, + "r2": 0.43462812900543213, + "score": -0.550823697016921 + } + }, + "time": "0:19:35.624704", + "diversity": { + "train": { + "mean_centered_corr": -0.02013956569135189, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0812845528125763 + }, + "val": { + "mean_centered_corr": -0.02026885561645031, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08767195791006088 + }, + "test": { + "mean_centered_corr": -0.019879044964909554, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08455239981412888 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 1, + 18, + 29, + 14, + 11, + 20, + 10, + 13, + 25, + 12, + 6, + 26 + ], + "head_selection_time": "0:00:00.359737" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ed541478f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5159634144292107, + "mae": 0.410226047039032, + "r2": 0.5010189414024353, + "score": -0.5159634144292107 + }, + "val": { + "rmse": 0.5593939165294649, + "mae": 0.4475388526916504, + "r2": 0.40991663932800293, + "score": -0.5593939165294649 + }, + "test": { + "rmse": 0.5608508784321002, + "mae": 0.45661959052085876, + "r2": 0.41385674476623535, + "score": -0.5608508784321002 + } + }, + "time": "0:16:46.550435", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.023743" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..a3c652359 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.25 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..f72692cdd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5088939009594995, + "mae": 0.40171560645103455, + "r2": 0.5145988464355469, + "score": -0.5088939009594995 + }, + "val": { + "rmse": 0.5558022325623276, + "mae": 0.44188886880874634, + "r2": 0.4174697995185852, + "score": -0.5558022325623276 + }, + "test": { + "rmse": 0.5512480144508224, + "mae": 0.4457772672176361, + "r2": 0.4337567090988159, + "score": -0.5512480144508224 + } + }, + "time": "0:16:46.550435", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2224a0377 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p25_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,191 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.25, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.25, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 2184, + "metrics": { + "train": { + "rmse": 0.5086880691661995, + "mae": 0.4017886221408844, + "r2": 0.5149915218353271, + "score": -0.5086880691661995 + }, + "val": { + "rmse": 0.5554104781133956, + "mae": 0.44194847345352173, + "r2": 0.4182906746864319, + "score": -0.5554104781133956 + }, + "test": { + "rmse": 0.5525622419967886, + "mae": 0.4474184215068817, + "r2": 0.4310535788536072, + "score": -0.5525622419967886 + } + }, + "time": "0:16:46.550435", + "diversity": { + "train": { + "mean_centered_corr": -0.019908307120203972, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07479876279830933 + }, + "val": { + "mean_centered_corr": -0.02006814442574978, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07624257355928421 + }, + "test": { + "mean_centered_corr": -0.019939016550779343, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07578995823860168 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 4, + 13, + 18, + 11, + 15, + 6, + 21, + 19, + 29, + 31, + 16, + 14, + 22 + ], + "head_selection_time": "0:00:00.397701" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7e807e060 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.516478064040768, + "mae": 0.40249156951904297, + "r2": 0.5000230073928833, + "score": -0.516478064040768 + }, + "val": { + "rmse": 0.5579574268466545, + "mae": 0.4429668188095093, + "r2": 0.41294336318969727, + "score": -0.5579574268466545 + }, + "test": { + "rmse": 0.5527197091141781, + "mae": 0.44649186730384827, + "r2": 0.4307292699813843, + "score": -0.5527197091141781 + } + }, + "time": "0:14:45.943513", + "diversity": { + "train": { + "mean_centered_corr": -0.020442524924874306, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06176675111055374 + }, + "val": { + "mean_centered_corr": -0.020368289202451706, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06270569562911987 + }, + "test": { + "mean_centered_corr": -0.020271461457014084, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06161132827401161 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023288" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0.toml new file mode 100644 index 000000000..e2f18e728 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0/report.json new file mode 100644 index 000000000..138ab14aa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5141129141438994, + "mae": 0.4034823179244995, + "r2": 0.504591703414917, + "score": -0.5141129141438994 + }, + "val": { + "rmse": 0.5560025757843153, + "mae": 0.44079530239105225, + "r2": 0.41704970598220825, + "score": -0.5560025757843153 + }, + "test": { + "rmse": 0.550425719096006, + "mae": 0.44393521547317505, + "r2": 0.43544477224349976, + "score": -0.550425719096006 + } + }, + "time": "0:14:45.943513", + "diversity": { + "train": { + "mean_centered_corr": -0.020442524924874306, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06176675111055374 + }, + "val": { + "mean_centered_corr": -0.020368289202451706, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06270569562911987 + }, + "test": { + "mean_centered_corr": -0.020271461457014084, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06161132827401161 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..75cd99ca0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0-greedy-heads-evaluation/0/report.json @@ -0,0 +1,187 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1365, + "metrics": { + "train": { + "rmse": 0.5144092503450841, + "mae": 0.40473756194114685, + "r2": 0.5040204524993896, + "score": -0.5144092503450841 + }, + "val": { + "rmse": 0.5553178831836687, + "mae": 0.44113633036613464, + "r2": 0.418484628200531, + "score": -0.5553178831836687 + }, + "test": { + "rmse": 0.5514062081885815, + "mae": 0.4456017017364502, + "r2": 0.4334316849708557, + "score": -0.5514062081885815 + } + }, + "time": "0:14:45.943513", + "diversity": { + "train": { + "mean_centered_corr": -0.020442524924874306, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06176675111055374 + }, + "val": { + "mean_centered_corr": -0.020368289202451706, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06270569562911987 + }, + "test": { + "mean_centered_corr": -0.020271461457014084, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06161132827401161 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 15, + 11, + 14, + 26, + 10, + 31, + 16, + 6 + ], + "head_selection_time": "0:00:00.258774" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e4887b5bf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-best-head-evaluation/0/report.json @@ -0,0 +1,179 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.502955286059696, + "mae": 0.3946622908115387, + "r2": 0.5258617401123047, + "score": -0.502955286059696 + }, + "val": { + "rmse": 0.5596610856119687, + "mae": 0.44277745485305786, + "r2": 0.4093528389930725, + "score": -0.5596610856119687 + }, + "test": { + "rmse": 0.5563749515907954, + "mae": 0.44843393564224243, + "r2": 0.4231749176979065, + "score": -0.5563749515907954 + } + }, + "time": "0:19:38.897747", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13 + ], + "head_selection_time": "0:00:00.023489" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0.toml new file mode 100644 index 000000000..58e4ac18e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0.toml @@ -0,0 +1,40 @@ +batch_size = 1024 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 0.5 +rla_extra_paths_freeze_fraction = 0.5 + +[data] +cache = true +path = "data/delivery-eta" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0025639267063470926 +weight_decay = 0.0507074044872309 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 2 +d_block = 752 +dropout = 0.18437573041970334 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 32 + +[bins] +n_bins = 81 diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0/report.json new file mode 100644 index 000000000..6a3c21869 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-evaluation/0/report.json @@ -0,0 +1,175 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4956110911684622, + "mae": 0.3902001976966858, + "r2": 0.5396074056625366, + "score": -0.4956110911684622 + }, + "val": { + "rmse": 0.5558360123700841, + "mae": 0.43872880935668945, + "r2": 0.41739898920059204, + "score": -0.5558360123700841 + }, + "test": { + "rmse": 0.5491424149361828, + "mae": 0.4410218894481659, + "r2": 0.43807417154312134, + "score": -0.5491424149361828 + } + }, + "time": "0:19:38.897747", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d2d3dff39 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n1em05_lr0p5_fr0p5-greedy-heads-evaluation/0/report.json @@ -0,0 +1,194 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 1024, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 0.5, + "rla_extra_paths_freeze_fraction": 0.5, + "data": { + "cache": true, + "path": "data/delivery-eta", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0025639267063470926, + "weight_decay": 0.0507074044872309 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 2, + "d_block": 752, + "dropout": 0.18437573041970334 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 32 + } + }, + "bins": { + "n_bins": 81 + } + }, + "n_parameters": 7647904, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 273, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 6980, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 752 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 752, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 752 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 0.5, + "extra_paths_freeze_fraction": 0.5, + "extra_paths_freeze_until_epoch": 16 + }, + "best_step": 3276, + "metrics": { + "train": { + "rmse": 0.4960975196274642, + "mae": 0.3916766345500946, + "r2": 0.5387032628059387, + "score": -0.4960975196274642 + }, + "val": { + "rmse": 0.5552555453020565, + "mae": 0.4401964545249939, + "r2": 0.4186151623725891, + "score": -0.5552555453020565 + }, + "test": { + "rmse": 0.551575730248866, + "mae": 0.44500499963760376, + "r2": 0.43308329582214355, + "score": -0.551575730248866 + } + }, + "time": "0:19:38.897747", + "diversity": { + "train": { + "mean_centered_corr": -0.020183589309453964, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0822177454829216 + }, + "val": { + "mean_centered_corr": -0.020031888037919998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08821859210729599 + }, + "test": { + "mean_centered_corr": -0.01992768421769142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08592649549245834 + } + }, + "chunk_size": null, + "eval_batch_size": 8192, + "heads": [ + 13, + 10, + 2, + 20, + 16, + 6, + 26, + 17, + 21, + 15, + 1, + 10, + 22, + 2, + 30, + 31 + ], + "head_selection_time": "0:00:00.511746" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/rescue_selected_configs.csv b/tabm_integrated/paper/exp/rescue_selected_configs.csv new file mode 100644 index 000000000..8278598c8 --- /dev/null +++ b/tabm_integrated/paper/exp/rescue_selected_configs.csv @@ -0,0 +1,6 @@ +dataset,module,final_variant,source_variant,source_wave,inference_mode,validation_metric,matched_validation_baseline,validation_delta,validation_percent_delta,validation_status,validation_score,source_config_path,seed0_result_path,confirm_3seed +delivery-eta,RLA,best_rla_only,deliv_rla_r4_n0p0001_lr0p5_fr0,rescue_delivery_individual,best-head,0.5572652691682626,0.5583881453279098,0.0011228761596472348,0.20109240660684746,validation_win,-0.5572652691682626,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-best-head-evaluation/0,True +delivery-eta,ESAM,best_esam_only,deliv_esam_rho0p001_sf0p25_adapterf,rescue_delivery_individual,mean,0.5558269510040955,0.5558749906446844,4.8039640588859456e-05,0.008642166205956623,validation_win,-0.5558269510040955,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0,True +delivery-eta,MFB,best_mfb_only,deliv_mfb_k0p975_sf0p5_numerical_only,rescue_delivery_individual,mean,0.5558048331442785,0.5558749906446844,7.015750040584301e-05,0.012621093157020213,validation_win,-0.5558048331442785,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0,True +delivery-eta,CF-FISD,best_cf_fisd_only,deliv_cf_l0p005_consensus_raw,rescue_delivery_individual,greedy-heads,0.5553348954063525,0.5553348954063525,0.0,0.0,close_no_validation_win,-0.5553348954063525,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0.toml,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-greedy-heads-evaluation/0,True +delivery-eta,combined,best_combined,deliv_comb_rla_cf,rescue_delivery_combined,best-head,0.5572652691682626,0.5583881453279098,0.0011228761596472348,0.20109240660684746,validation_win,-0.5572652691682626,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-best-head-evaluation/0,True diff --git a/tabm_integrated/paper/exp/rescue_targets.csv b/tabm_integrated/paper/exp/rescue_targets.csv new file mode 100644 index 000000000..a5c900274 --- /dev/null +++ b/tabm_integrated/paper/exp/rescue_targets.csv @@ -0,0 +1,6 @@ +priority,dataset,module,final_variant,current_source_variant,current_inference_mode,metric,direction,matched_baseline_test_mean,current_test_mean,current_absolute_delta,current_percent_delta,current_status,rescue_reason +0,delivery-eta,RLA,best_rla_only,rla_rank2_noise0.001,greedy-heads,RMSE,lower,0.5511744515458483,0.5514065614560619,-0.00023210991021360705,-0.04211187756664361,loss,matched_baseline_loss +0,delivery-eta,ESAM,best_esam_only,esam_only,greedy-heads,RMSE,lower,0.5511744515458483,0.5511479242060323,2.652733981600708e-05,0.004812875441088992,weak_win,delivery_eta_priority +0,delivery-eta,MFB,best_mfb_only,mfb_keep0.7,greedy-heads,RMSE,lower,0.5511744515458483,0.551366165184366,-0.00019171363851766543,-0.03478275126504446,loss,matched_baseline_loss +0,delivery-eta,CF-FISD,best_cf_fisd_only,cf_fisd_lambda0.05,greedy-heads,RMSE,lower,0.5511744515458483,0.5524340881807381,-0.0012596366348898158,-0.22853683282252707,loss,matched_baseline_loss +0,delivery-eta,combined,best_combined,rla_esam,greedy-heads,RMSE,lower,0.5511744515458483,0.5523362750771902,-0.0011618235313418657,-0.21079052704336418,loss,matched_baseline_loss diff --git a/tabm_integrated/paper/exp/selected_integrated_configs.csv b/tabm_integrated/paper/exp/selected_integrated_configs.csv index 474f775f8..393b807a6 100644 --- a/tabm_integrated/paper/exp/selected_integrated_configs.csv +++ b/tabm_integrated/paper/exp/selected_integrated_configs.csv @@ -1,31 +1,31 @@ -dataset,final_variant,source_variant,source_wave,inference_mode,validation_metric,validation_score,source_config_path -sberbank-housing,baseline_plr,baseline_plr,smoke,mean,,,exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0.toml -sberbank-housing,best_rla_only,rla_rank2_noise0.001,sweeps,greedy-heads,0.2328613038082034,-0.2328613038082034,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0.toml -sberbank-housing,best_esam_only,esam_rho0.0025,sweeps,greedy-heads,0.23295802298295287,-0.23295802298295287,exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0.toml -sberbank-housing,best_mfb_only,mfb_keep0.8,sweeps,greedy-heads,0.23237596587359466,-0.23237596587359466,exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0.toml -sberbank-housing,best_cf_fisd_only,cf_fisd_only,smoke,greedy-heads,0.23327114512588593,-0.23327114512588593,exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0.toml -sberbank-housing,best_combined,mfb_cf_fisd,sweeps,greedy-heads,0.23320900639668657,-0.23320900639668657,exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0.toml -ecom-offers,baseline_plr,baseline_plr,smoke,mean,,,exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0.toml -ecom-offers,best_rla_only,rla_rank2_noise0.0001,sweeps,best-head,0.6396541901648656,0.6396541901648656,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0.toml -ecom-offers,best_esam_only,esam_rho0.001,sweeps,greedy-heads,0.641845068859757,0.641845068859757,exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0.toml -ecom-offers,best_mfb_only,mfb_keep0.7,sweeps,best-head,0.6547032702847417,0.6547032702847417,exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0.toml -ecom-offers,best_cf_fisd_only,cf_fisd_lambda0.05,sweeps,greedy-heads,0.6419149054445072,0.6419149054445072,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0.toml -ecom-offers,best_combined,mfb_cf_fisd,sweeps,greedy-heads,0.6506783465179156,0.6506783465179156,exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0.toml -homesite-insurance,baseline_plr,baseline_plr,smoke,mean,,,exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0.toml -homesite-insurance,best_rla_only,rla_rank4_noise1e-05,sweeps,greedy-heads,0.9591329763748824,0.9591329763748824,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0.toml -homesite-insurance,best_esam_only,esam_only,smoke,greedy-heads,0.958991133281215,0.958991133281215,exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0.toml -homesite-insurance,best_mfb_only,mfb_keep0.95,sweeps,greedy-heads,0.9592957095194116,0.9592957095194116,exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0.toml -homesite-insurance,best_cf_fisd_only,cf_fisd_only,smoke,greedy-heads,0.9594844321379927,0.9594844321379927,exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0.toml -homesite-insurance,best_combined,mfb_cf_fisd,sweeps,greedy-heads,0.9597570226838343,0.9597570226838343,exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0.toml -cooking-time,baseline_plr,baseline_plr,smoke,mean,,,exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0.toml -cooking-time,best_rla_only,rla_rank4_noise0.001,sweeps,greedy-heads,0.4624107384246987,-0.4624107384246987,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0.toml -cooking-time,best_esam_only,esam_rho0.0025,sweeps,greedy-heads,0.46246732196836127,-0.46246732196836127,exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0.toml -cooking-time,best_mfb_only,mfb_keep0.8,sweeps,greedy-heads,0.4624822561623469,-0.4624822561623469,exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0.toml -cooking-time,best_cf_fisd_only,cf_fisd_lambda0.05,sweeps,greedy-heads,0.4628482268025946,-0.4628482268025946,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0.toml -cooking-time,best_combined,rla_esam,sweeps,greedy-heads,0.4624234187671886,-0.4624234187671886,exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0.toml -delivery-eta,baseline_plr,baseline_plr,smoke,mean,,,exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0.toml -delivery-eta,best_rla_only,rla_rank2_noise0.001,sweeps,greedy-heads,0.5552720495549479,-0.5552720495549479,exp/integrated/sweeps/delivery-eta/rla_rank2_noise0.001-evaluation/0.toml -delivery-eta,best_esam_only,esam_only,smoke,greedy-heads,0.5551513290724005,-0.5551513290724005,exp/integrated/smoke/delivery-eta/esam_only-evaluation/0.toml -delivery-eta,best_mfb_only,mfb_keep0.7,sweeps,greedy-heads,0.5551008108444078,-0.5551008108444078,exp/integrated/sweeps/delivery-eta/mfb_keep0.7-evaluation/0.toml -delivery-eta,best_cf_fisd_only,cf_fisd_lambda0.05,sweeps,greedy-heads,0.5556691040453957,-0.5556691040453957,exp/integrated/sweeps/delivery-eta/cf_fisd_lambda0.05-evaluation/0.toml -delivery-eta,best_combined,rla_esam,sweeps,greedy-heads,0.5552988041877751,-0.5552988041877751,exp/integrated/sweeps/delivery-eta/rla_esam-evaluation/0.toml +dataset,final_variant,source_variant,source_wave,inference_mode,validation_metric,validation_score,source_config_path,matched_validation_baseline,validation_delta,validation_percent_delta,validation_status +sberbank-housing,baseline_plr,baseline_plr,smoke,mean,,,exp/integrated/smoke/sberbank-housing/baseline_plr-evaluation/0.toml,,,, +sberbank-housing,best_rla_only,rla_rank2_noise0.001,sweeps,greedy-heads,0.2328613038082034,-0.2328613038082034,exp/integrated/sweeps/sberbank-housing/rla_rank2_noise0.001-evaluation/0.toml,,,, +sberbank-housing,best_esam_only,esam_rho0.0025,sweeps,greedy-heads,0.23295802298295287,-0.23295802298295287,exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0.toml,,,, +sberbank-housing,best_mfb_only,mfb_keep0.8,sweeps,greedy-heads,0.23237596587359466,-0.23237596587359466,exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0.toml,,,, +sberbank-housing,best_cf_fisd_only,cf_fisd_only,smoke,greedy-heads,0.23327114512588593,-0.23327114512588593,exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0.toml,,,, +sberbank-housing,best_combined,mfb_cf_fisd,sweeps,greedy-heads,0.23320900639668657,-0.23320900639668657,exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0.toml,,,, +ecom-offers,baseline_plr,baseline_plr,smoke,mean,,,exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0.toml,,,, +ecom-offers,best_rla_only,rla_rank2_noise0.0001,sweeps,best-head,0.6396541901648656,0.6396541901648656,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0.toml,,,, +ecom-offers,best_esam_only,esam_rho0.001,sweeps,greedy-heads,0.641845068859757,0.641845068859757,exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0.toml,,,, +ecom-offers,best_mfb_only,mfb_keep0.7,sweeps,best-head,0.6547032702847417,0.6547032702847417,exp/integrated/sweeps/ecom-offers/mfb_keep0.7-evaluation/0.toml,,,, +ecom-offers,best_cf_fisd_only,cf_fisd_lambda0.05,sweeps,greedy-heads,0.6419149054445072,0.6419149054445072,exp/integrated/sweeps/ecom-offers/cf_fisd_lambda0.05-evaluation/0.toml,,,, +ecom-offers,best_combined,mfb_cf_fisd,sweeps,greedy-heads,0.6506783465179156,0.6506783465179156,exp/integrated/sweeps/ecom-offers/mfb_cf_fisd-evaluation/0.toml,,,, +homesite-insurance,baseline_plr,baseline_plr,smoke,mean,,,exp/integrated/smoke/homesite-insurance/baseline_plr-evaluation/0.toml,,,, +homesite-insurance,best_rla_only,rla_rank4_noise1e-05,sweeps,greedy-heads,0.9591329763748824,0.9591329763748824,exp/integrated/sweeps/homesite-insurance/rla_rank4_noise1e-05-evaluation/0.toml,,,, +homesite-insurance,best_esam_only,esam_only,smoke,greedy-heads,0.958991133281215,0.958991133281215,exp/integrated/smoke/homesite-insurance/esam_only-evaluation/0.toml,,,, +homesite-insurance,best_mfb_only,mfb_keep0.95,sweeps,greedy-heads,0.9592957095194116,0.9592957095194116,exp/integrated/sweeps/homesite-insurance/mfb_keep0.95-evaluation/0.toml,,,, +homesite-insurance,best_cf_fisd_only,cf_fisd_only,smoke,greedy-heads,0.9594844321379927,0.9594844321379927,exp/integrated/smoke/homesite-insurance/cf_fisd_only-evaluation/0.toml,,,, +homesite-insurance,best_combined,mfb_cf_fisd,sweeps,greedy-heads,0.9597570226838343,0.9597570226838343,exp/integrated/sweeps/homesite-insurance/mfb_cf_fisd-evaluation/0.toml,,,, +cooking-time,baseline_plr,baseline_plr,smoke,mean,,,exp/integrated/smoke/cooking-time/baseline_plr-evaluation/0.toml,,,, +cooking-time,best_rla_only,rla_rank4_noise0.001,sweeps,greedy-heads,0.4624107384246987,-0.4624107384246987,exp/integrated/sweeps/cooking-time/rla_rank4_noise0.001-evaluation/0.toml,,,, +cooking-time,best_esam_only,esam_rho0.0025,sweeps,greedy-heads,0.46246732196836127,-0.46246732196836127,exp/integrated/sweeps/cooking-time/esam_rho0.0025-evaluation/0.toml,,,, +cooking-time,best_mfb_only,mfb_keep0.8,sweeps,greedy-heads,0.4624822561623469,-0.4624822561623469,exp/integrated/sweeps/cooking-time/mfb_keep0.8-evaluation/0.toml,,,, +cooking-time,best_cf_fisd_only,cf_fisd_lambda0.05,sweeps,greedy-heads,0.4628482268025946,-0.4628482268025946,exp/integrated/sweeps/cooking-time/cf_fisd_lambda0.05-evaluation/0.toml,,,, +cooking-time,best_combined,rla_esam,sweeps,greedy-heads,0.4624234187671886,-0.4624234187671886,exp/integrated/sweeps/cooking-time/rla_esam-evaluation/0.toml,,,, +delivery-eta,baseline_plr,baseline_plr,smoke,mean,,,exp/integrated/smoke/delivery-eta/baseline_plr-evaluation/0.toml,,,, +delivery-eta,best_rla_only,deliv_rla_r4_n0p0001_lr0p5_fr0,rescue_delivery_individual,best-head,0.5572652691682626,-0.5572652691682626,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_rla_r4_n0p0001_lr0p5_fr0-evaluation/0.toml,0.5583881453279098,0.0011228761596472348,0.20109240660684746,validation_win +delivery-eta,best_esam_only,deliv_esam_rho0p001_sf0p25_adapterf,rescue_delivery_individual,mean,0.5558269510040955,-0.5558269510040955,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_esam_rho0p001_sf0p25_adapterf-evaluation/0.toml,0.5558749906446844,4.8039640588859456e-05,0.008642166205956623,validation_win +delivery-eta,best_mfb_only,deliv_mfb_k0p975_sf0p5_numerical_only,rescue_delivery_individual,mean,0.5558048331442785,-0.5558048331442785,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_mfb_k0p975_sf0p5_numerical_only-evaluation/0.toml,0.5558749906446844,7.015750040584301e-05,0.012621093157020213,validation_win +delivery-eta,best_cf_fisd_only,deliv_cf_l0p005_consensus_raw,rescue_delivery_individual,greedy-heads,0.5553348954063525,-0.5553348954063525,exp/integrated/rescue_delivery_individual/delivery-eta/deliv_cf_l0p005_consensus_raw-evaluation/0.toml,0.5553348954063525,0.0,0.0,close_no_validation_win +delivery-eta,best_combined,deliv_comb_rla_cf,rescue_delivery_combined,best-head,0.5572652691682626,-0.5572652691682626,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_cf-evaluation/0.toml,0.5583881453279098,0.0011228761596472348,0.20109240660684746,validation_win diff --git a/tabm_integrated/tools/aggregate_integrated_results.py b/tabm_integrated/tools/aggregate_integrated_results.py index c39da8918..5bfbc330f 100644 --- a/tabm_integrated/tools/aggregate_integrated_results.py +++ b/tabm_integrated/tools/aggregate_integrated_results.py @@ -19,6 +19,7 @@ AUDIT_PATH = PAPER / 'exp' / 'final_integrated_audit.csv' REPORT_PATH = PAPER.parent / 'FINAL_EXPERIMENT_REPORT.md' SELECTED_PATH = PAPER / 'exp' / 'selected_integrated_configs.csv' +RESCUE_SELECTED_PATH = PAPER / 'exp' / 'rescue_selected_configs.csv' FINAL_MANIFEST = EXP_ROOT / 'manifest_final.txt' DATASET_ORDER = ['sberbank-housing', 'ecom-offers', 'homesite-insurance', 'cooking-time', 'delivery-eta'] @@ -238,6 +239,9 @@ def summarize_final() -> list[dict[str, Any]]: baseline_rs = sorted(baseline_grouped.get((dataset, selected_inference), []), key=lambda r: r['seed']) baseline_invalid = len(baseline_rs) != 3 or any(r['failure'] for r in baseline_rs) val_mean = statistics.mean([r['validation_metric'] for r in rs]) if rs else float('nan') + baseline_val_values = [r['validation_metric'] for r in baseline_rs] + base_val_mean = statistics.mean(baseline_val_values) if baseline_val_values else float('nan') + val_delta, val_pct = signed_delta(val_mean, base_val_mean, direction) if rs and baseline_rs else (float('nan'), float('nan')) test_values = [r['test_metric'] for r in rs] test_mean = statistics.mean(test_values) if test_values else float('nan') test_std = statistics.stdev(test_values) if len(test_values) > 1 else 0.0 @@ -260,6 +264,9 @@ def summarize_final() -> list[dict[str, Any]]: 'direction': direction, 'metric_direction': direction, 'validation_metric': val_mean, + 'matched_validation_baseline': base_val_mean, + 'matched_validation_delta': val_delta, + 'matched_validation_percent_delta': val_pct, 'test_metric': test_mean, 'mean': test_mean, 'std': test_std, @@ -326,7 +333,7 @@ def write_report(summary_rows: list[dict[str, Any]]) -> None: lines += [ '## Matched-inference correction', '', - 'The earlier mean-baseline comparison changed when every variant was compared with the matching baseline inference mode. Per the rescue protocol stop condition, no new rescue sweep is claimed in this report.', + 'The earlier mean-baseline comparison changed when every variant was compared with the matching baseline inference mode. Rescue sweeps are selected against matched validation baselines only.', '', '| dataset | variant | inference | mean-baseline status | matched-baseline status | mean-baseline delta | matched delta |', '|---|---|---|---|---|---:|---:|', @@ -339,15 +346,26 @@ def write_report(summary_rows: list[dict[str, Any]]) -> None: lines += ['## Final 3-seed results', ''] lines += [ - '| dataset | task | variant | selected config | metric | direction | inference | matched baseline mean ± std | result mean ± std | delta | % delta | n | status | config path | result path |', - '|---|---|---|---|---|---|---|---:|---:|---:|---:|---:|---|---|---|', + '| dataset | task | variant | selected config | metric | direction | inference | matched validation baseline | validation metric | matched baseline test mean ± std | result test mean ± std | delta | % delta | n | status | config path | result path |', + '|---|---|---|---|---|---|---|---:|---:|---:|---:|---:|---:|---:|---|---|---|', ] for r in summary_rows: baseline_mean_std = f"{fmt(r['matched_baseline_mean'])} ± {fmt(r['matched_baseline_std'])}" mean_std = f"{fmt(r['mean'])} ± {fmt(r['std'])}" lines.append( - f"| {r['dataset']} | {r['task_type']} | {r['variant']} | {r['source_variant']} | {r['metric']} | {r['direction']} | {r['inference_mode']} | {baseline_mean_std} | {mean_std} | {fmt(r['absolute_delta'])} | {fmt(r['percent_delta'])} | {r['n_seeds']} | {r['status']} | `{r['config_path']}` | `{r['result_path']}` |" + f"| {r['dataset']} | {r['task_type']} | {r['variant']} | {r['source_variant']} | {r['metric']} | {r['direction']} | {r['inference_mode']} | {fmt(r['matched_validation_baseline'])} | {fmt(r['validation_metric'])} | {baseline_mean_std} | {mean_std} | {fmt(r['absolute_delta'])} | {fmt(r['percent_delta'])} | {r['n_seeds']} | {r['status']} | `{r['config_path']}` | `{r['result_path']}` |" ) + if RESCUE_SELECTED_PATH.exists(): + lines += ['', '## Rescue validation selections', ''] + lines += [ + '| dataset | final variant | selected rescue config | inference | validation metric | matched validation baseline | validation delta | validation status | confirmed |', + '|---|---|---|---|---:|---:|---:|---|---|', + ] + with RESCUE_SELECTED_PATH.open() as f: + for row in csv.DictReader(f): + lines.append( + f"| {row['dataset']} | {row['final_variant']} | {row['source_variant']} | {row['inference_mode']} | {row['validation_metric']} | {row['matched_validation_baseline']} | {row['validation_delta']} | {row['validation_status']} | {row['confirm_3seed']} |" + ) lines += ['', '## Validation-selected configs', ''] if SELECTED_PATH.exists(): lines += ['| dataset | final variant | selected sweep variant | inference | validation metric |', '|---|---|---|---|---:|'] diff --git a/tabm_integrated/tools/delivery_rescue.py b/tabm_integrated/tools/delivery_rescue.py new file mode 100644 index 000000000..abf1d9531 --- /dev/null +++ b/tabm_integrated/tools/delivery_rescue.py @@ -0,0 +1,361 @@ +from __future__ import annotations + +import argparse +import csv +import itertools +import json +import math +import statistics +import sys +import tomllib +from collections import Counter, defaultdict +from copy import deepcopy +from pathlib import Path +from typing import Any + +sys.dont_write_bytecode = True + +import tomli_w + +TOOLS = Path(__file__).resolve().parent +PAPER = TOOLS.parent / 'paper' +EXP_ROOT = PAPER / 'exp' / 'integrated' +TARGETS_PATH = PAPER / 'exp' / 'rescue_targets.csv' +RESCUE_SELECTED_PATH = PAPER / 'exp' / 'rescue_selected_configs.csv' +SELECTED_PATH = PAPER / 'exp' / 'selected_integrated_configs.csv' +IND_MANIFEST = EXP_ROOT / 'manifest_delivery_rescue_individual.txt' +COMB_MANIFEST = EXP_ROOT / 'manifest_delivery_rescue_combined.txt' +FINAL_MANIFEST = EXP_ROOT / 'manifest_delivery_rescue_final.txt' +DATASET = 'delivery-eta' + +sys.path.insert(0, str(TOOLS)) +import generate_integrated_configs as gen # noqa: E402 +import aggregate_integrated_results as agg # noqa: E402 + +MODULE_VARIANTS = { + 'RLA': 'best_rla_only', + 'ESAM': 'best_esam_only', + 'MFB': 'best_mfb_only', + 'CF-FISD': 'best_cf_fisd_only', + 'combined': 'best_combined', +} + + +def ftag(x: float | int | bool) -> str: + if isinstance(x, bool): + return 't' if x else 'f' + if isinstance(x, int): + return str(x) + if x == 0: + return '0' + return f'{x:g}'.replace('-', 'm').replace('.', 'p') + + +def start_epoch(cfg: dict[str, Any], frac: float) -> int: + budget = int(cfg['n_epochs']) if int(cfg['n_epochs']) > 0 else 2 * int(cfg['patience']) + return int(math.ceil(budget * frac)) + + +def write_csv(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fields: list[str] = [] + for row in rows: + for key in row: + if key not in fields: + fields.append(key) + with path.open('w', newline='') as f: + writer = csv.DictWriter(f, fieldnames=fields, lineterminator='\n') + writer.writeheader() + writer.writerows(rows) + + +def read_csv(path: Path) -> list[dict[str, str]]: + with path.open(newline='') as f: + return list(csv.DictReader(f)) + + +def write_config(wave: str, variant: str, cfg: dict[str, Any], seed: int = 0) -> Path: + cfg = deepcopy(cfg) + cfg['seed'] = seed + if cfg.get('model', {}).get('mfb'): + cfg['model']['mfb']['mask_seed'] = seed + path = EXP_ROOT / wave / DATASET / f'{variant}-evaluation' / f'{seed}.toml' + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(tomli_w.dumps(cfg)) + return path + + +def apply_rla(cfg: dict[str, Any], rank: int, noise: float, lr: float, freeze: float) -> None: + gen.apply_rla(cfg, rank=rank, noise=noise) + cfg['model']['rla_first_only'] = False + cfg['rla_adapter_lr_multiplier'] = lr + cfg['rla_extra_paths_freeze_fraction'] = freeze + + +def apply_esam(cfg: dict[str, Any], rho: float, frac: float, adapter_only: bool) -> None: + gen.apply_esam(cfg, rho=rho) + cfg['esam_adapter_only'] = adapter_only + cfg['esam_start_epoch'] = start_epoch(cfg, frac) + + +def apply_mfb(cfg: dict[str, Any], keep: float, frac: float, group_mode: str) -> None: + gen.apply_mfb(cfg, keep=keep) + cfg['model']['mfb']['start_epoch'] = start_epoch(cfg, frac) + cfg['model']['mfb']['start_fraction'] = frac + cfg['model']['mfb']['group_mode'] = group_mode + cfg['model']['mfb']['categorical_handling'] = 'no_cat_drop' + + +def apply_cf(cfg: dict[str, Any], lam: float, mode: str) -> None: + gen.apply_cf_fisd(cfg, DATASET, lam=lam) + cf = cfg['cf_fisd'] + cf['start_epoch'] = start_epoch(cfg, 0.5) + cf['start_fraction'] = 0.5 + cf['mode'] = mode + if mode == 'softmax': + cf['variant'] = 'softmax' + elif mode == 'consensus_raw': + cf['variant'] = 'raw' + k = int(cfg['model']['k']) + cf['member_groups'] = {name: list(range(k)) for name in cf['teacher_names']} + else: + raise ValueError(mode) + + +def generate_targets() -> None: + rows = read_csv(PAPER / 'exp' / 'final_integrated_summary.csv') + out = [] + for row in rows: + if row['dataset'] != DATASET or row['variant'] == 'baseline_plr': + continue + module = {v: k for k, v in MODULE_VARIANTS.items()}[row['variant']] + out.append({ + 'priority': 0, + 'dataset': DATASET, + 'module': module, + 'final_variant': row['variant'], + 'current_source_variant': row['source_variant'], + 'current_inference_mode': row['inference_mode'], + 'metric': row['metric'], + 'direction': row['direction'], + 'matched_baseline_test_mean': row['matched_baseline_mean'], + 'current_test_mean': row['mean'], + 'current_absolute_delta': row['absolute_delta'], + 'current_percent_delta': row['percent_delta'], + 'current_status': row['status'], + 'rescue_reason': 'delivery_eta_priority' if row['status'] in {'clear_win', 'weak_win'} else 'matched_baseline_loss', + }) + order = {'RLA': 0, 'ESAM': 1, 'MFB': 2, 'CF-FISD': 3, 'combined': 4} + out.sort(key=lambda r: order[r['module']]) + write_csv(TARGETS_PATH, out) + print(f'{TARGETS_PATH.relative_to(PAPER)} rows={len(out)} counts={dict(Counter(r["module"] for r in out))}') + + +def generate_individual() -> None: + paths: list[Path] = [] + for rank, noise, lr, freeze in itertools.product([1, 2, 4], [0.0, 1e-5, 1e-4], [0.25, 0.5], [0.0, 0.5]): + cfg = gen.base_config(DATASET, 0) + apply_rla(cfg, rank, noise, lr, freeze) + paths.append(write_config('rescue_delivery_individual', f'deliv_rla_r{rank}_n{ftag(noise)}_lr{ftag(lr)}_fr{ftag(freeze)}', cfg)) + for rho, frac, adapter_only in itertools.product([0.00025, 0.0005, 0.001], [0.25, 0.5], [True, False]): + cfg = gen.base_config(DATASET, 0) + apply_esam(cfg, rho, frac, adapter_only) + paths.append(write_config('rescue_delivery_individual', f'deliv_esam_rho{ftag(rho)}_sf{ftag(frac)}_adapter{ftag(adapter_only)}', cfg)) + for keep, frac, group_mode in itertools.product([0.975, 0.99], [0.5, 0.75], ['numerical_only', 'per_member']): + cfg = gen.base_config(DATASET, 0) + apply_mfb(cfg, keep, frac, group_mode) + paths.append(write_config('rescue_delivery_individual', f'deliv_mfb_k{ftag(keep)}_sf{ftag(frac)}_{group_mode}', cfg)) + for lam, mode in itertools.product([0.005, 0.01, 0.02], ['consensus_raw', 'softmax']): + cfg = gen.base_config(DATASET, 0) + apply_cf(cfg, lam, mode) + paths.append(write_config('rescue_delivery_individual', f'deliv_cf_l{ftag(lam)}_{mode}', cfg)) + IND_MANIFEST.write_text('\n'.join(str(p.relative_to(PAPER)) for p in paths) + '\n') + print(f'{IND_MANIFEST.relative_to(PAPER)} configs={len(paths)}') + + +def family(variant: str) -> str | None: + if variant.startswith('deliv_rla_'): + return 'RLA' + if variant.startswith('deliv_esam_'): + return 'ESAM' + if variant.startswith('deliv_mfb_'): + return 'MFB' + if variant.startswith('deliv_cf_'): + return 'CF-FISD' + if variant.startswith('deliv_comb_'): + return 'combined' + return None + + +def delta(value: float, baseline: float, direction: str) -> tuple[float, float]: + if direction == 'lower': + d = baseline - value + return d, 100.0 * d / baseline if baseline else float('nan') + d = value - baseline + return d, 100.0 * d / abs(baseline) if baseline else float('nan') + + +def seed0_baselines(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + out = {} + for row in rows: + if row['wave'] == 'final' and row['dataset'] == DATASET and row['variant'] == 'baseline_plr' and row['seed'] == 0 and not row['failure']: + out[row['inference_mode']] = row + missing = [x for x in ['mean', 'best-head', 'greedy-heads'] if x not in out] + if missing: + raise RuntimeError(f'missing delivery seed-0 baselines: {missing}') + return out + + +def candidate_rows(wave: str) -> list[dict[str, Any]]: + rows = agg.collect_rows() + bases = seed0_baselines(rows) + out = [] + for row in rows: + if row['wave'] != wave or row['dataset'] != DATASET or row['seed'] != 0 or row['failure']: + continue + mod = family(row['variant']) + if mod is None: + continue + base = bases[row['inference_mode']] + d, pct = delta(row['validation_metric'], base['validation_metric'], row['direction']) + enriched = dict(row) + enriched['module'] = mod + enriched['final_variant'] = MODULE_VARIANTS[mod] + enriched['matched_validation_baseline'] = base['validation_metric'] + enriched['validation_delta'] = d + enriched['validation_percent_delta'] = pct + enriched['validation_status'] = 'validation_win' if d > 1e-12 else ('close_no_validation_win' if pct >= -0.1 else 'no_validation_win') + out.append(enriched) + return out + + +def best_by_module(wave: str, modules: list[str]) -> dict[str, dict[str, Any]]: + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + for row in candidate_rows(wave): + grouped[row['module']].append(row) + best = {} + for mod in modules: + if not grouped.get(mod): + raise RuntimeError(f'no completed candidates for {mod} in {wave}') + best[mod] = max(grouped[mod], key=lambda r: (r['validation_delta'], r['validation_score'])) + return best + + +def copy_rla(dst: dict[str, Any], src: dict[str, Any]) -> None: + for key in ['rla_rank', 'rla_first_only', 'rla_additive', 'rla_init', 'rla_base_preserve_noise']: + if key in src['model']: + dst['model'][key] = src['model'][key] + for key in ['rla_adapter_lr_multiplier', 'rla_extra_paths_freeze_fraction']: + if key in src: + dst[key] = src[key] + + +def copy_esam(dst: dict[str, Any], src: dict[str, Any]) -> None: + for key in ['use_esam', 'esam_rho', 'esam_eps', 'esam_adapter_only', 'esam_memberwise', 'esam_warmup_epochs', 'esam_start_epoch', 'esam_end_epoch', 'esam_log_diagnostics', 'esam_diagnostics_every']: + if key in src: + dst[key] = src[key] + + +def copy_cf(dst: dict[str, Any], src: dict[str, Any]) -> None: + dst['cf_fisd'] = deepcopy(src['cf_fisd']) + + +def generate_combined() -> None: + best = best_by_module('rescue_delivery_individual', ['RLA', 'ESAM', 'MFB', 'CF-FISD']) + cfgs = {mod: tomllib.loads((PAPER / row['config_path']).read_text()) for mod, row in best.items()} + combos = { + 'deliv_comb_rla_esam': ('RLA', 'ESAM'), + 'deliv_comb_rla_cf': ('RLA', 'CF-FISD'), + 'deliv_comb_esam_cf': ('ESAM', 'CF-FISD'), + 'deliv_comb_rla_esam_cf': ('RLA', 'ESAM', 'CF-FISD'), + } + paths = [] + for variant, mods in combos.items(): + cfg = gen.base_config(DATASET, 0) + if 'RLA' in mods: + copy_rla(cfg, cfgs['RLA']) + if 'ESAM' in mods: + copy_esam(cfg, cfgs['ESAM']) + if 'CF-FISD' in mods: + copy_cf(cfg, cfgs['CF-FISD']) + paths.append(write_config('rescue_delivery_combined', variant, cfg)) + COMB_MANIFEST.write_text('\n'.join(str(p.relative_to(PAPER)) for p in paths) + '\n') + print(f'{COMB_MANIFEST.relative_to(PAPER)} configs={len(paths)}') + + +def select_and_make_final() -> None: + selected = best_by_module('rescue_delivery_individual', ['RLA', 'ESAM', 'MFB', 'CF-FISD']) + selected.update(best_by_module('rescue_delivery_combined', ['combined'])) + rows = [] + for mod in ['RLA', 'ESAM', 'MFB', 'CF-FISD', 'combined']: + row = selected[mod] + rows.append({ + 'dataset': DATASET, + 'module': mod, + 'final_variant': MODULE_VARIANTS[mod], + 'source_variant': row['variant'], + 'source_wave': row['wave'], + 'inference_mode': row['inference_mode'], + 'validation_metric': row['validation_metric'], + 'matched_validation_baseline': row['matched_validation_baseline'], + 'validation_delta': row['validation_delta'], + 'validation_percent_delta': row['validation_percent_delta'], + 'validation_status': row['validation_status'], + 'validation_score': row['validation_score'], + 'source_config_path': row['config_path'], + 'seed0_result_path': row['result_path'], + 'confirm_3seed': True, + }) + write_csv(RESCUE_SELECTED_PATH, rows) + + selection = read_csv(SELECTED_PATH) + repl = {(r['dataset'], r['final_variant']): r for r in rows} + updated = [] + for row in selection: + key = (row['dataset'], row['final_variant']) + if key in repl: + r = repl[key] + new = dict(row) + new.update({ + 'source_variant': r['source_variant'], + 'source_wave': r['source_wave'], + 'inference_mode': r['inference_mode'], + 'validation_metric': str(r['validation_metric']), + 'validation_score': str(r['validation_score']), + 'matched_validation_baseline': str(r['matched_validation_baseline']), + 'validation_delta': str(r['validation_delta']), + 'validation_percent_delta': str(r['validation_percent_delta']), + 'validation_status': r['validation_status'], + 'source_config_path': r['source_config_path'], + }) + updated.append(new) + else: + updated.append(row) + write_csv(SELECTED_PATH, updated) + + final_paths = [] + for r in rows: + cfg = tomllib.loads((PAPER / r['source_config_path']).read_text()) + for seed in [0, 1, 2]: + final_paths.append(write_config('final', r['final_variant'], cfg, seed)) + FINAL_MANIFEST.write_text('\n'.join(str(p.relative_to(PAPER)) for p in final_paths) + '\n') + print(f'{RESCUE_SELECTED_PATH.relative_to(PAPER)} rows={len(rows)}') + print(f'{FINAL_MANIFEST.relative_to(PAPER)} configs={len(final_paths)}') + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('stage', choices=['targets', 'generate-individual', 'generate-combined', 'select-final']) + args = parser.parse_args() + if args.stage == 'targets': + generate_targets() + elif args.stage == 'generate-individual': + generate_individual() + elif args.stage == 'generate-combined': + generate_combined() + elif args.stage == 'select-final': + select_and_make_final() + + +if __name__ == '__main__': + main() From d5d07c8511c853a4e27ee24925fb17f318f2e8e0 Mon Sep 17 00:00:00 2001 From: abhipray-chavan <53275745+abhipray-chavan@users.noreply.github.com> Date: Sun, 26 Apr 2026 23:03:43 +0000 Subject: [PATCH 5/7] Improve integrated TabM results --- tabm_integrated/FINAL_EXPERIMENT_REPORT.md | 7 +- .../paper/exp/final_integrated_audit.csv | 567 +++++++++++++- .../paper/exp/final_integrated_summary.csv | 2 +- .../0/report.json | 77 +- .../1/report.json | 77 +- .../2/report.json | 77 +- .../best_combined-evaluation/0.toml | 11 +- .../best_combined-evaluation/0/report.json | 73 +- .../best_combined-evaluation/1.toml | 11 +- .../best_combined-evaluation/1/report.json | 73 +- .../best_combined-evaluation/2.toml | 11 +- .../best_combined-evaluation/2/report.json | 73 +- .../0/report.json | 88 +-- .../1/report.json | 92 ++- .../2/report.json | 89 ++- ...anifest_sberbank_combined_rescue_final.txt | 6 + ...anifest_sberbank_combined_rescue_seed0.txt | 177 +++++ .../0/DONE | 0 .../0/report.json | 652 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 648 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 652 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 648 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 657 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 652 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 648 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 661 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 652 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 648 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 661 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 652 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 648 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 652 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 648 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 657 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 652 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 648 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 661 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 652 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 648 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 661 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 652 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 648 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 652 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 648 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 657 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 652 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 648 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 661 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 652 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 648 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 661 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 652 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 648 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 652 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 648 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 657 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 652 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 648 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 661 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 652 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 648 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 661 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 664 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 663 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 667 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 667 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 663 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 668 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 667 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 663 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 661 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 665 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 666 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 663 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 662 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 667 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 665 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 665 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 664 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 663 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 667 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 667 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 664 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 663 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 673 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 668 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 665 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 666 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 665 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 666 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 663 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 664 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 666 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 658 ++++++++++++++++ .../0.toml | 52 ++ .../0/DONE | 0 .../0/report.json | 654 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 665 ++++++++++++++++ .../0/DONE | 0 .../0/report.json | 207 +++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 203 +++++ .../0/DONE | 0 .../0/report.json | 217 ++++++ .../0/DONE | 0 .../0/report.json | 207 +++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 203 +++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0/DONE | 0 .../0/report.json | 207 +++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 203 +++++ .../0/DONE | 0 .../0/report.json | 219 ++++++ .../0/DONE | 0 .../0/report.json | 207 +++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 203 +++++ .../0/DONE | 0 .../0/report.json | 214 ++++++ .../0/DONE | 0 .../0/report.json | 207 +++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 203 +++++ .../0/DONE | 0 .../0/report.json | 217 ++++++ .../0/DONE | 0 .../0/report.json | 207 +++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 203 +++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0/DONE | 0 .../0/report.json | 207 +++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 203 +++++ .../0/DONE | 0 .../0/report.json | 219 ++++++ .../0/DONE | 0 .../0/report.json | 207 +++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 203 +++++ .../0/DONE | 0 .../0/report.json | 214 ++++++ .../0/DONE | 0 .../0/report.json | 207 +++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 203 +++++ .../0/DONE | 0 .../0/report.json | 217 ++++++ .../0/DONE | 0 .../0/report.json | 207 +++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 203 +++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0/DONE | 0 .../0/report.json | 207 +++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 203 +++++ .../0/DONE | 0 .../0/report.json | 219 ++++++ .../0/DONE | 0 .../0/report.json | 207 +++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 203 +++++ .../0/DONE | 0 .../0/report.json | 214 ++++++ .../0/DONE | 0 .../0/report.json | 207 +++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 203 +++++ .../0/DONE | 0 .../0/report.json | 217 ++++++ .../0/DONE | 0 .../0/report.json | 207 +++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 203 +++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0/DONE | 0 .../0/report.json | 207 +++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 203 +++++ .../0/DONE | 0 .../0/report.json | 219 ++++++ .../0/DONE | 0 .../0/report.json | 207 +++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 203 +++++ .../0/DONE | 0 .../0/report.json | 214 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 224 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 221 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 224 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 224 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 221 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 220 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 223 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 220 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 220 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 218 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 221 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 223 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 219 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 225 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 223 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 219 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 220 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 218 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 219 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 220 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 220 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 223 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 225 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 221 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 226 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 220 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 219 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 221 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 218 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 224 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 217 ++++++ .../0/DONE | 0 .../0/report.json | 213 ++++++ .../0.toml | 51 ++ .../0/DONE | 0 .../0/report.json | 209 ++++++ .../0/DONE | 0 .../0/report.json | 219 ++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 691 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 693 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 693 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 692 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 696 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 695 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 699 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 705 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 698 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 691 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 693 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 693 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 692 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 696 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 695 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 699 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 705 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 698 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 691 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 693 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 693 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 692 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 696 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 695 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 699 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 705 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 698 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 700 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 702 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 697 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 698 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 699 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 701 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 703 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 698 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 708 ++++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 703 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 701 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 700 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 700 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 704 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 699 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 701 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 704 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 702 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 698 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 698 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 707 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 701 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 699 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 698 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 700 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 703 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 705 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 701 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 700 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 699 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 699 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 697 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 699 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 699 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 704 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 702 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 699 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 698 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 697 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 699 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 700 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 707 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 701 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 700 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 704 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 703 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 700 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 698 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 702 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 701 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 701 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 697 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 704 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 694 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 690 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 702 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../1/DONE | 0 .../1/report.json | 688 +++++++++++++++++ .../2/DONE | 0 .../2/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../1.toml | 65 ++ .../1/DONE | 0 .../1/report.json | 684 +++++++++++++++++ .../2.toml | 65 ++ .../2/DONE | 0 .../2/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 692 +++++++++++++++++ .../1/DONE | 0 .../1/report.json | 703 +++++++++++++++++ .../2/DONE | 0 .../2/report.json | 696 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 688 +++++++++++++++++ .../1/DONE | 0 .../1/report.json | 688 +++++++++++++++++ .../2/DONE | 0 .../2/report.json | 688 +++++++++++++++++ .../0.toml | 65 ++ .../0/DONE | 0 .../0/report.json | 684 +++++++++++++++++ .../1.toml | 65 ++ .../1/DONE | 0 .../1/report.json | 684 +++++++++++++++++ .../2.toml | 65 ++ .../2/DONE | 0 .../2/report.json | 684 +++++++++++++++++ .../0/DONE | 0 .../0/report.json | 692 +++++++++++++++++ .../1/DONE | 0 .../1/report.json | 703 +++++++++++++++++ .../2/DONE | 0 .../2/report.json | 696 +++++++++++++++++ .../sberbank_combined_rescue_confirmed.csv | 3 + .../exp/sberbank_combined_rescue_selected.csv | 21 + .../paper/exp/selected_integrated_configs.csv | 2 +- .../tools/sberbank_combined_rescue.py | 266 +++++++ 1302 files changed, 318096 insertions(+), 336 deletions(-) create mode 100644 tabm_integrated/paper/exp/integrated/manifest_sberbank_combined_rescue_final.txt create mode 100644 tabm_integrated/paper/exp/integrated/manifest_sberbank_combined_rescue_seed0.txt create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2.toml create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/1/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/1/report.json create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/2/DONE create mode 100644 tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/2/report.json create mode 100644 tabm_integrated/paper/exp/sberbank_combined_rescue_confirmed.csv create mode 100644 tabm_integrated/paper/exp/sberbank_combined_rescue_selected.csv create mode 100644 tabm_integrated/tools/sberbank_combined_rescue.py diff --git a/tabm_integrated/FINAL_EXPERIMENT_REPORT.md b/tabm_integrated/FINAL_EXPERIMENT_REPORT.md index 12144d59b..c11fe9602 100644 --- a/tabm_integrated/FINAL_EXPERIMENT_REPORT.md +++ b/tabm_integrated/FINAL_EXPERIMENT_REPORT.md @@ -10,6 +10,7 @@ The earlier mean-baseline comparison changed when every variant was compared wit | dataset | variant | inference | mean-baseline status | matched-baseline status | mean-baseline delta | matched delta | |---|---|---|---|---|---:|---:| | sberbank-housing | best_esam_only | greedy-heads | weak_win | loss | 1.17846e-05 | -0.000301017 | +| sberbank-housing | best_combined | best-head | loss | weak_win | -0.0106618 | 0.00512063 | | ecom-offers | best_rla_only | best-head | clear_win | loss | 0.00838888 | -0.00124693 | | ecom-offers | best_esam_only | greedy-heads | clear_win | weak_win | 0.0103616 | 4.90059e-05 | | ecom-offers | best_cf_fisd_only | greedy-heads | clear_win | loss | 0.0101468 | -0.000165703 | @@ -35,7 +36,7 @@ The earlier mean-baseline comparison changed when every variant was compared wit | sberbank-housing | regression | best_esam_only | esam_rho0.0025 | RMSE | lower | greedy-heads | 0.233757 | 0.233815 | 0.234105 ± 0.00278385 | 0.234406 ± 0.00401172 | -0.000301017 | -0.128582 | 3 | loss | `exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2` | | sberbank-housing | regression | best_mfb_only | mfb_keep0.8 | RMSE | lower | greedy-heads | 0.233757 | 0.233219 | 0.234105 ± 0.00278385 | 0.2351 ± 0.00395661 | -0.000994573 | -0.424841 | 3 | loss | `exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2` | | sberbank-housing | regression | best_cf_fisd_only | cf_fisd_only | RMSE | lower | greedy-heads | 0.233757 | 0.234079 | 0.234105 ± 0.00278385 | 0.233093 ± 0.000585082 | 0.001012 | 0.432283 | 3 | weak_win | `exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2` | -| sberbank-housing | regression | best_combined | mfb_cf_fisd | RMSE | lower | greedy-heads | 0.233757 | 0.233292 | 0.234105 ± 0.00278385 | 0.234979 ± 0.00337063 | -0.000873715 | -0.373215 | 3 | loss | `exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2` | +| sberbank-housing | regression | best_combined | sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001 | RMSE | lower | best-head | 0.240171 | 0.238874 | 0.2502 ± 0.0136007 | 0.24508 ± 0.00811205 | 0.00512063 | 2.04661 | 3 | weak_win | `exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml` | `exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0;exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1;exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2` | | ecom-offers | binclass | baseline_plr | baseline_plr | AUROC | higher | mean | 0.62143 | 0.62143 | 0.590058 ± 0.000970364 | 0.590058 ± 0.000970364 | 0 | 0 | 3 | baseline | `exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml` | `exp/integrated/final/ecom-offers/baseline_plr-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2` | | ecom-offers | binclass | best_rla_only | rla_rank2_noise0.0001 | AUROC | higher | best-head | 0.635345 | 0.634828 | 0.599694 ± 0.00357854 | 0.598447 ± 0.00279735 | -0.00124693 | -0.207927 | 3 | loss | `exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2` | | ecom-offers | binclass | best_esam_only | esam_rho0.001 | AUROC | higher | greedy-heads | 0.63659 | 0.636656 | 0.600371 ± 0.00344598 | 0.60042 ± 0.00349335 | 4.90059e-05 | 0.0081626 | 3 | weak_win | `exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml` | `exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2` | @@ -80,7 +81,7 @@ The earlier mean-baseline comparison changed when every variant was compared wit | sberbank-housing | best_esam_only | esam_rho0.0025 | greedy-heads | 0.23295802298295287 | | sberbank-housing | best_mfb_only | mfb_keep0.8 | greedy-heads | 0.23237596587359466 | | sberbank-housing | best_cf_fisd_only | cf_fisd_only | greedy-heads | 0.23327114512588593 | -| sberbank-housing | best_combined | mfb_cf_fisd | greedy-heads | 0.23320900639668657 | +| sberbank-housing | best_combined | sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001 | best-head | 0.23560357747729638 | | ecom-offers | baseline_plr | baseline_plr | mean | | | ecom-offers | best_rla_only | rla_rank2_noise0.0001 | best-head | 0.6396541901648656 | | ecom-offers | best_esam_only | esam_rho0.001 | greedy-heads | 0.641845068859757 | @@ -112,4 +113,4 @@ The earlier mean-baseline comparison changed when every variant was compared wit - `best_esam_only`: ecom-offers, homesite-insurance, cooking-time - `best_mfb_only`: ecom-offers, cooking-time, delivery-eta - `best_cf_fisd_only`: sberbank-housing, homesite-insurance -- `best_combined`: ecom-offers, homesite-insurance, cooking-time, delivery-eta +- `best_combined`: sberbank-housing, ecom-offers, homesite-insurance, cooking-time, delivery-eta diff --git a/tabm_integrated/paper/exp/final_integrated_audit.csv b/tabm_integrated/paper/exp/final_integrated_audit.csv index eacfa652c..b9f49e6cc 100644 --- a/tabm_integrated/paper/exp/final_integrated_audit.csv +++ b/tabm_integrated/paper/exp/final_integrated_audit.csv @@ -584,9 +584,9 @@ final,sberbank-housing,best_mfb_only,2,RMSE,rmse,lower,0.23605616345434166,-0.23 final,sberbank-housing,best_cf_fisd_only,0,RMSE,rmse,lower,0.23523413233394516,-0.23523413233394516,0.235149605552611,mean,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0,False,666 final,sberbank-housing,best_cf_fisd_only,1,RMSE,rmse,lower,0.23639255622691643,-0.23639255622691643,0.2324256095265638,mean,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1,False,814 final,sberbank-housing,best_cf_fisd_only,2,RMSE,rmse,lower,0.2359054032023901,-0.2359054032023901,0.23245970634502372,mean,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2,False,518 -final,sberbank-housing,best_combined,0,RMSE,rmse,lower,0.23508488120210985,-0.23508488120210985,0.23568789402876444,mean,exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_combined-evaluation/0,False,888 -final,sberbank-housing,best_combined,1,RMSE,rmse,lower,0.23530641494725335,-0.23530641494725335,0.23169352906704774,mean,exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_combined-evaluation/1,False,592 -final,sberbank-housing,best_combined,2,RMSE,rmse,lower,0.2356153727014129,-0.2356153727014129,0.2360324823167294,mean,exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_combined-evaluation/2,False,1036 +final,sberbank-housing,best_combined,0,RMSE,rmse,lower,0.235050071622046,-0.235050071622046,0.23184680318718037,mean,exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_combined-evaluation/0,False,666 +final,sberbank-housing,best_combined,1,RMSE,rmse,lower,0.23618618194018487,-0.23618618194018487,0.23253236276211406,mean,exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_combined-evaluation/1,False,1184 +final,sberbank-housing,best_combined,2,RMSE,rmse,lower,0.23592642069691294,-0.23592642069691294,0.2326191697888986,mean,exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_combined-evaluation/2,False,888 final,sberbank-housing,baseline_plr,0,RMSE,rmse,lower,0.23955454428788905,-0.23955454428788905,0.26588264760298186,best-head,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/0.toml,exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/0,False,814 final,sberbank-housing,baseline_plr,2,RMSE,rmse,lower,0.23944956044068919,-0.23944956044068919,0.24308412455882378,best-head,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/2.toml,exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/2,False,666 final,sberbank-housing,baseline_plr,1,RMSE,rmse,lower,0.24150787640249508,-0.24150787640249508,0.2416341058127328,best-head,exp/integrated/final/sberbank-housing/baseline_plr-evaluation/1.toml,exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/1,False,962 @@ -617,12 +617,12 @@ final,sberbank-housing,best_cf_fisd_only,1,RMSE,rmse,lower,0.24061445857155628,- final,sberbank-housing,best_cf_fisd_only,0,RMSE,rmse,lower,0.23327114512588593,-0.23327114512588593,0.23367141178994547,greedy-heads,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0,False,666 final,sberbank-housing,best_cf_fisd_only,2,RMSE,rmse,lower,0.23432768503071347,-0.23432768503071347,0.2325014731401514,greedy-heads,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2,False,518 final,sberbank-housing,best_cf_fisd_only,1,RMSE,rmse,lower,0.23463880474397839,-0.23463880474397839,0.2331063026442073,greedy-heads,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1,False,814 -final,sberbank-housing,best_combined,0,RMSE,rmse,lower,0.23765498863853576,-0.23765498863853576,0.2536206620016002,best-head,exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0,False,888 -final,sberbank-housing,best_combined,1,RMSE,rmse,lower,0.2374959967300924,-0.2374959967300924,0.24912023719870632,best-head,exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1,False,592 -final,sberbank-housing,best_combined,2,RMSE,rmse,lower,0.23866731604984137,-0.23866731604984137,0.2357692652387237,best-head,exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2,False,1036 -final,sberbank-housing,best_combined,0,RMSE,rmse,lower,0.23320900639668657,-0.23320900639668657,0.23362227205150313,greedy-heads,exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0,False,888 -final,sberbank-housing,best_combined,1,RMSE,rmse,lower,0.2332021853813015,-0.2332021853813015,0.2324977398321572,greedy-heads,exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1,False,592 -final,sberbank-housing,best_combined,2,RMSE,rmse,lower,0.23346459476611428,-0.23346459476611428,0.23881630936614148,greedy-heads,exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2,False,1036 +final,sberbank-housing,best_combined,0,RMSE,rmse,lower,0.23560357747729638,-0.23560357747729638,0.2389191702722735,best-head,exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0,False,666 +final,sberbank-housing,best_combined,1,RMSE,rmse,lower,0.24254141605050564,-0.24254141605050564,0.24204913155172003,best-head,exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1,False,1184 +final,sberbank-housing,best_combined,2,RMSE,rmse,lower,0.23847619665119374,-0.23847619665119374,0.2542706881867437,best-head,exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2,False,888 +final,sberbank-housing,best_combined,0,RMSE,rmse,lower,0.2324412121462238,-0.2324412121462238,0.23820740696857753,greedy-heads,exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml,exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0,False,666 +final,sberbank-housing,best_combined,1,RMSE,rmse,lower,0.23397077829789442,-0.23397077829789442,0.23276530465893142,greedy-heads,exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml,exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1,False,1184 +final,sberbank-housing,best_combined,2,RMSE,rmse,lower,0.23411027740292373,-0.23411027740292373,0.24496736545125247,greedy-heads,exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2,False,888 final,ecom-offers,baseline_plr,0,AUROC,roc-auc,higher,0.6227054517511603,0.6227054517511603,0.591151981311678,mean,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0,False,321 final,ecom-offers,baseline_plr,2,AUROC,roc-auc,higher,0.6212503263524413,0.6212503263524413,0.5893011087345269,mean,exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml,exp/integrated/final/ecom-offers/baseline_plr-evaluation/2,False,321 final,ecom-offers,baseline_plr,1,AUROC,roc-auc,higher,0.6203331748306973,0.6203331748306973,0.5897210540664094,mean,exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml,exp/integrated/final/ecom-offers/baseline_plr-evaluation/1,False,428 @@ -1037,3 +1037,552 @@ rescue_delivery_combined,delivery-eta,deliv_comb_rla_esam,0,RMSE,rmse,lower,0.55 rescue_delivery_combined,delivery-eta,deliv_comb_rla_esam,0,RMSE,rmse,lower,0.5550917643214267,-0.5550917643214267,0.5526470209655567,greedy-heads,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam-greedy-heads-evaluation/0,False,2730 rescue_delivery_combined,delivery-eta,deliv_comb_rla_esam_cf,0,RMSE,rmse,lower,0.5591176650506209,-0.5591176650506209,0.5583917212440949,best-head,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-best-head-evaluation/0,False,2730 rescue_delivery_combined,delivery-eta,deliv_comb_rla_esam_cf,0,RMSE,rmse,lower,0.5550917643214267,-0.5550917643214267,0.5526470209655567,greedy-heads,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-evaluation/0.toml,exp/integrated/rescue_delivery_combined/delivery-eta/deliv_comb_rla_esam_cf-greedy-heads-evaluation/0,False,2730 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p001,0,RMSE,rmse,lower,0.23516710262665957,-0.23516710262665957,0.2392893867678542,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p005,0,RMSE,rmse,lower,0.23541557264636437,-0.23541557264636437,0.23955821427368978,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p01,0,RMSE,rmse,lower,0.23516078988949235,-0.23516078988949235,0.24165682950771084,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p02,0,RMSE,rmse,lower,0.23537261362228468,-0.23537261362228468,0.23578993148696667,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p001,0,RMSE,rmse,lower,0.23516710262665957,-0.23516710262665957,0.2392893867678542,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p005,0,RMSE,rmse,lower,0.23541557264636437,-0.23541557264636437,0.23955821427368978,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p01,0,RMSE,rmse,lower,0.23516078988949235,-0.23516078988949235,0.24165682950771084,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p02,0,RMSE,rmse,lower,0.23537261362228468,-0.23537261362228468,0.23578993148696667,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p001,0,RMSE,rmse,lower,0.23516710262665957,-0.23516710262665957,0.2392893867678542,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p005,0,RMSE,rmse,lower,0.23541557264636437,-0.23541557264636437,0.23955821427368978,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p01,0,RMSE,rmse,lower,0.23516078988949235,-0.23516078988949235,0.24165682950771084,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p02,0,RMSE,rmse,lower,0.23537261362228468,-0.23537261362228468,0.23578993148696667,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p001,0,RMSE,rmse,lower,0.23516710262665957,-0.23516710262665957,0.2392893867678542,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p005,0,RMSE,rmse,lower,0.23541557264636437,-0.23541557264636437,0.23955821427368978,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p01,0,RMSE,rmse,lower,0.23516078988949235,-0.23516078988949235,0.24165682950771084,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p02,0,RMSE,rmse,lower,0.23537261362228468,-0.23537261362228468,0.23578993148696667,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p001,0,RMSE,rmse,lower,0.23516964509902982,-0.23516964509902982,0.23928895864366434,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p005,0,RMSE,rmse,lower,0.23541076200529618,-0.23541076200529618,0.2395584319829453,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p01,0,RMSE,rmse,lower,0.23516104335271928,-0.23516104335271928,0.24165658285763553,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p02,0,RMSE,rmse,lower,0.23537506682607195,-0.23537506682607195,0.23579029486809283,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p001,0,RMSE,rmse,lower,0.23523322965051088,-0.23523322965051088,0.23438947156097584,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p005,0,RMSE,rmse,lower,0.23519216180885402,-0.23519216180885402,0.23499174827687583,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p01,0,RMSE,rmse,lower,0.23534793771825302,-0.23534793771825302,0.2358765820307155,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p02,0,RMSE,rmse,lower,0.23499558463666298,-0.23499558463666298,0.23934769790965568,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p001,0,RMSE,rmse,lower,0.23492518063329523,-0.23492518063329523,0.23863885185928208,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p005,0,RMSE,rmse,lower,0.23501775344788706,-0.23501775344788706,0.23276117546354494,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p01,0,RMSE,rmse,lower,0.23514059116024372,-0.23514059116024372,0.23749547910151964,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p02,0,RMSE,rmse,lower,0.23506069808417399,-0.23506069808417399,0.24070191052163564,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p001,0,RMSE,rmse,lower,0.23558290278789318,-0.23558290278789318,0.24195079618816864,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p005,0,RMSE,rmse,lower,0.23531810633502007,-0.23531810633502007,0.2327283473545216,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p01,0,RMSE,rmse,lower,0.23480153213840635,-0.23480153213840635,0.23483096909658374,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p02,0,RMSE,rmse,lower,0.23510019641837185,-0.23510019641837185,0.23935828921252758,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p001,0,RMSE,rmse,lower,0.23516964509902982,-0.23516964509902982,0.23928895864366434,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p005,0,RMSE,rmse,lower,0.23541076200529618,-0.23541076200529618,0.2395584319829453,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p01,0,RMSE,rmse,lower,0.23516104335271928,-0.23516104335271928,0.24165658285763553,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p02,0,RMSE,rmse,lower,0.23537506682607195,-0.23537506682607195,0.23579029486809283,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p001,0,RMSE,rmse,lower,0.23532450983472006,-0.23532450983472006,0.2376596833156001,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p005,0,RMSE,rmse,lower,0.23472067454979642,-0.23472067454979642,0.23644700488851306,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p01,0,RMSE,rmse,lower,0.23505473908329258,-0.23505473908329258,0.23281250512039894,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p02,0,RMSE,rmse,lower,0.23511142271763394,-0.23511142271763394,0.23473994135203347,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p001,0,RMSE,rmse,lower,0.2350763793606662,-0.2350763793606662,0.23832022238784234,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p005,0,RMSE,rmse,lower,0.2350520527289723,-0.2350520527289723,0.23495368248701565,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p01,0,RMSE,rmse,lower,0.23466412671675002,-0.23466412671675002,0.2324620941369229,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p02,0,RMSE,rmse,lower,0.23532386870106714,-0.23532386870106714,0.23728542485175996,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p001,0,RMSE,rmse,lower,0.2349816418639806,-0.2349816418639806,0.24136844697035015,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p005,0,RMSE,rmse,lower,0.23549047304076126,-0.23549047304076126,0.24128326690664834,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p01,0,RMSE,rmse,lower,0.23543217967973734,-0.23543217967973734,0.23786624351296862,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p02,0,RMSE,rmse,lower,0.23514095554484465,-0.23514095554484465,0.23359667766611852,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p00025,0,RMSE,rmse,lower,0.23484986994593354,-0.23484986994593354,0.23462197483665095,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p0005,0,RMSE,rmse,lower,0.23510992538054096,-0.23510992538054096,0.24175341990615093,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p001,0,RMSE,rmse,lower,0.23467774706155475,-0.23467774706155475,0.23716614057842658,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p0025,0,RMSE,rmse,lower,0.23474623366768377,-0.23474623366768377,0.23936563514587833,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p00025,0,RMSE,rmse,lower,0.23484986994593354,-0.23484986994593354,0.23462197483665095,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p0005,0,RMSE,rmse,lower,0.23510992538054096,-0.23510992538054096,0.24175341990615093,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p001,0,RMSE,rmse,lower,0.23467774706155475,-0.23467774706155475,0.23716614057842658,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p0025,0,RMSE,rmse,lower,0.23474623366768377,-0.23474623366768377,0.23936563514587833,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p00025,0,RMSE,rmse,lower,0.23484986994593354,-0.23484986994593354,0.23462197483665095,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p0005,0,RMSE,rmse,lower,0.23510992538054096,-0.23510992538054096,0.24175341990615093,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p001,0,RMSE,rmse,lower,0.23467774706155475,-0.23467774706155475,0.23716614057842658,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p0025,0,RMSE,rmse,lower,0.23474623366768377,-0.23474623366768377,0.23936563514587833,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p00025,0,RMSE,rmse,lower,0.23484986994593354,-0.23484986994593354,0.23462197483665095,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p0005,0,RMSE,rmse,lower,0.23510992538054096,-0.23510992538054096,0.24175341990615093,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p001,0,RMSE,rmse,lower,0.23467774706155475,-0.23467774706155475,0.23716614057842658,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p0025,0,RMSE,rmse,lower,0.23474623366768377,-0.23474623366768377,0.23936563514587833,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p00025,0,RMSE,rmse,lower,0.23499185132041592,-0.23499185132041592,0.2394094259267782,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0,False,962 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p0005,0,RMSE,rmse,lower,0.2346688971022881,-0.2346688971022881,0.23375485551605374,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p001,0,RMSE,rmse,lower,0.23475735787864294,-0.23475735787864294,0.23961932837948463,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p0025,0,RMSE,rmse,lower,0.23456885744734607,-0.23456885744734607,0.23654787235061092,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p00025,0,RMSE,rmse,lower,0.23516112255992166,-0.23516112255992166,0.23211789488202353,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p0005,0,RMSE,rmse,lower,0.23464263099787872,-0.23464263099787872,0.23482863711969398,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p001,0,RMSE,rmse,lower,0.23461772747586743,-0.23461772747586743,0.23292056846137055,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p0025,0,RMSE,rmse,lower,0.23469026342966967,-0.23469026342966967,0.23333669987162928,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p00025,0,RMSE,rmse,lower,0.23496530422551093,-0.23496530422551093,0.23600846732737993,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p0005,0,RMSE,rmse,lower,0.23480699780789907,-0.23480699780789907,0.23801010369895165,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p001,0,RMSE,rmse,lower,0.2356879335438401,-0.2356879335438401,0.23192085632684412,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p0025,0,RMSE,rmse,lower,0.2352761982909868,-0.2352761982909868,0.2333683170052332,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p00025,0,RMSE,rmse,lower,0.23478980709452008,-0.23478980709452008,0.2332225120575295,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p0005,0,RMSE,rmse,lower,0.23527326904017234,-0.23527326904017234,0.24074289728147005,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p001,0,RMSE,rmse,lower,0.23507535721736414,-0.23507535721736414,0.2433693260742304,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p0025,0,RMSE,rmse,lower,0.23446964895916758,-0.23446964895916758,0.23505021426230255,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p00025,0,RMSE,rmse,lower,0.23541561220714985,-0.23541561220714985,0.23578183426225294,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p0005,0,RMSE,rmse,lower,0.23501861733133902,-0.23501861733133902,0.23373303710107815,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p001,0,RMSE,rmse,lower,0.23496360777291245,-0.23496360777291245,0.23914605469445158,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p0025,0,RMSE,rmse,lower,0.23463987641889242,-0.23463987641889242,0.23470931052134267,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p00025,0,RMSE,rmse,lower,0.23507776598207258,-0.23507776598207258,0.2438242655494171,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p0005,0,RMSE,rmse,lower,0.23477109182279635,-0.23477109182279635,0.2340687021877646,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p001,0,RMSE,rmse,lower,0.23488388446999084,-0.23488388446999084,0.23845471649958624,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p0025,0,RMSE,rmse,lower,0.23480782280356366,-0.23480782280356366,0.23834319169461882,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p00025,0,RMSE,rmse,lower,0.23513454704634784,-0.23513454704634784,0.2392125690267148,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p0005,0,RMSE,rmse,lower,0.2354849520464804,-0.2354849520464804,0.23141729611827572,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p001,0,RMSE,rmse,lower,0.23554663270650014,-0.23554663270650014,0.24084128499653581,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p0025,0,RMSE,rmse,lower,0.23546283513255054,-0.23546283513255054,0.23448563984262677,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0,False,740 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p00025,0,RMSE,rmse,lower,0.234735553294385,-0.234735553294385,0.23656269916289197,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p0005,0,RMSE,rmse,lower,0.2353183042208091,-0.2353183042208091,0.2326811298212561,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p001,0,RMSE,rmse,lower,0.23492211221645545,-0.23492211221645545,0.23523946918101388,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p0025,0,RMSE,rmse,lower,0.23496063498765302,-0.23496063498765302,0.2373818009282627,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001,0,RMSE,rmse,lower,0.2357328737026148,-0.2357328737026148,0.2351772485906963,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005,0,RMSE,rmse,lower,0.2351519581095747,-0.2351519581095747,0.2408751648791966,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0,False,1184 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01,0,RMSE,rmse,lower,0.2353263382433216,-0.2353263382433216,0.2351845825591179,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001,0,RMSE,rmse,lower,0.2347666170730829,-0.2347666170730829,0.23399684917333807,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005,0,RMSE,rmse,lower,0.2351252944190024,-0.2351252944190024,0.23697534320527558,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01,0,RMSE,rmse,lower,0.23512438339576403,-0.23512438339576403,0.2316782780837399,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.235050071622046,-0.235050071622046,0.23184680318718037,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005,0,RMSE,rmse,lower,0.23505585640746507,-0.23505585640746507,0.23276235981389973,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01,0,RMSE,rmse,lower,0.23504784483792887,-0.23504784483792887,0.23465037063607563,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001,0,RMSE,rmse,lower,0.2357328737026148,-0.2357328737026148,0.2351772485906963,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005,0,RMSE,rmse,lower,0.2351519581095747,-0.2351519581095747,0.2408751648791966,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0,False,1184 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01,0,RMSE,rmse,lower,0.2353263382433216,-0.2353263382433216,0.2351845825591179,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001,0,RMSE,rmse,lower,0.2347666170730829,-0.2347666170730829,0.23399684917333807,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005,0,RMSE,rmse,lower,0.2351252944190024,-0.2351252944190024,0.23697534320527558,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01,0,RMSE,rmse,lower,0.23512438339576403,-0.23512438339576403,0.2316782780837399,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.235050071622046,-0.235050071622046,0.23184680318718037,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005,0,RMSE,rmse,lower,0.23505585640746507,-0.23505585640746507,0.23276235981389973,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.23504784483792887,-0.23504784483792887,0.23465037063607563,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001,0,RMSE,rmse,lower,0.2357328737026148,-0.2357328737026148,0.2351772485906963,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005,0,RMSE,rmse,lower,0.2351519581095747,-0.2351519581095747,0.2408751648791966,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0,False,1184 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01,0,RMSE,rmse,lower,0.2353263382433216,-0.2353263382433216,0.2351845825591179,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001,0,RMSE,rmse,lower,0.2347666170730829,-0.2347666170730829,0.23399684917333807,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005,0,RMSE,rmse,lower,0.2351252944190024,-0.2351252944190024,0.23697534320527558,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01,0,RMSE,rmse,lower,0.23512438339576403,-0.23512438339576403,0.2316782780837399,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001,0,RMSE,rmse,lower,0.235050071622046,-0.235050071622046,0.23184680318718037,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005,0,RMSE,rmse,lower,0.23505585640746507,-0.23505585640746507,0.23276235981389973,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01,0,RMSE,rmse,lower,0.23504784483792887,-0.23504784483792887,0.23465037063607563,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001,0,RMSE,rmse,lower,0.23493513883666117,-0.23493513883666117,0.2324465730511325,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005,0,RMSE,rmse,lower,0.2350836134726586,-0.2350836134726586,0.23409832677668932,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01,0,RMSE,rmse,lower,0.23493006464565203,-0.23493006464565203,0.23367776476223437,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001,0,RMSE,rmse,lower,0.23512809084220013,-0.23512809084220013,0.2355998696072441,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005,0,RMSE,rmse,lower,0.23518098688395506,-0.23518098688395506,0.23184570253369863,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01,0,RMSE,rmse,lower,0.23473991754724038,-0.23473991754724038,0.23324559206850415,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.23468678716863678,-0.23468678716863678,0.23217868905016578,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005,0,RMSE,rmse,lower,0.23531282668072034,-0.23531282668072034,0.2329136030473919,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01,0,RMSE,rmse,lower,0.23489533520890316,-0.23489533520890316,0.23252367146541786,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001,0,RMSE,rmse,lower,0.2349283758671495,-0.2349283758671495,0.23427005643589985,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005,0,RMSE,rmse,lower,0.23481945176098018,-0.23481945176098018,0.23330914213736356,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01,0,RMSE,rmse,lower,0.235230434477205,-0.235230434477205,0.23369840873687428,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001,0,RMSE,rmse,lower,0.23518726739965692,-0.23518726739965692,0.23268638913916373,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005,0,RMSE,rmse,lower,0.23483364211643792,-0.23483364211643792,0.23156642561711185,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01,0,RMSE,rmse,lower,0.23457506700663386,-0.23457506700663386,0.2318295295556057,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.2351439260504182,-0.2351439260504182,0.23458613581335708,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005,0,RMSE,rmse,lower,0.2346577290091983,-0.2346577290091983,0.23333821657015275,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.23515957801466283,-0.23515957801466283,0.23255229143999848,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001,0,RMSE,rmse,lower,0.234848704054223,-0.234848704054223,0.23316213384908455,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005,0,RMSE,rmse,lower,0.23504563388193753,-0.23504563388193753,0.2337165723322532,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01,0,RMSE,rmse,lower,0.2351138707244049,-0.2351138707244049,0.23346224116260836,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001,0,RMSE,rmse,lower,0.2350451108570483,-0.2350451108570483,0.23328519811438406,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005,0,RMSE,rmse,lower,0.23490809371963584,-0.23490809371963584,0.23175764135614477,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01,0,RMSE,rmse,lower,0.23465687967262416,-0.23465687967262416,0.23108455049001858,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001,0,RMSE,rmse,lower,0.23496993373476788,-0.23496993373476788,0.23346087685954336,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005,0,RMSE,rmse,lower,0.23492098632735753,-0.23492098632735753,0.2411043186306478,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01,0,RMSE,rmse,lower,0.23479386888101392,-0.23479386888101392,0.23419678588244902,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001,0,RMSE,rmse,lower,0.23544191074619497,-0.23544191074619497,0.23248160420213973,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005,0,RMSE,rmse,lower,0.23458071263974148,-0.23458071263974148,0.23412343670770672,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01,0,RMSE,rmse,lower,0.23471617503170125,-0.23471617503170125,0.23405948700639934,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001,0,RMSE,rmse,lower,0.23558367762733304,-0.23558367762733304,0.2306278293973631,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005,0,RMSE,rmse,lower,0.23476767229656906,-0.23476767229656906,0.23170820829575262,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01,0,RMSE,rmse,lower,0.2351196777064841,-0.2351196777064841,0.23285365668962787,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.23556640917246102,-0.23556640917246102,0.23394963288342588,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005,0,RMSE,rmse,lower,0.23502662197192856,-0.23502662197192856,0.23487478850600307,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01,0,RMSE,rmse,lower,0.2349593190227014,-0.2349593190227014,0.23427020750192581,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001,0,RMSE,rmse,lower,0.23522375918391628,-0.23522375918391628,0.23361716934488255,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005,0,RMSE,rmse,lower,0.23474049679651998,-0.23474049679651998,0.23271906309028628,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01,0,RMSE,rmse,lower,0.23523273079765322,-0.23523273079765322,0.231476592513396,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001,0,RMSE,rmse,lower,0.23522175576712184,-0.23522175576712184,0.2392933877459825,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005,0,RMSE,rmse,lower,0.2351262925795415,-0.2351262925795415,0.23336620188006213,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01,0,RMSE,rmse,lower,0.23509613200143026,-0.23509613200143026,0.23336054282826887,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.2346724530004704,-0.2346724530004704,0.23472572946107031,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005,0,RMSE,rmse,lower,0.23514965307924723,-0.23514965307924723,0.23159657135010447,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.23526017405140137,-0.23526017405140137,0.23237540477719482,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001,0,RMSE,rmse,lower,0.2350854992177386,-0.2350854992177386,0.23319390245153368,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005,0,RMSE,rmse,lower,0.23460196790457225,-0.23460196790457225,0.23160625447220415,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01,0,RMSE,rmse,lower,0.23525231195767085,-0.23525231195767085,0.23376059267196012,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001,0,RMSE,rmse,lower,0.2352393029012561,-0.2352393029012561,0.23369680670218232,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005,0,RMSE,rmse,lower,0.23530708779168627,-0.23530708779168627,0.23121293389473055,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01,0,RMSE,rmse,lower,0.234867262459557,-0.234867262459557,0.23225197071547168,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001,0,RMSE,rmse,lower,0.23505533340532211,-0.23505533340532211,0.2334740169208309,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005,0,RMSE,rmse,lower,0.23498608081258596,-0.23498608081258596,0.23378104608060551,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01,0,RMSE,rmse,lower,0.23478007280066537,-0.23478007280066537,0.2351072792694515,mean,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p02,0,RMSE,rmse,lower,0.23785345573254446,-0.23785345573254446,0.23747813004889556,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p02,0,RMSE,rmse,lower,0.23340410361335404,-0.23340410361335404,0.2382911617531396,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p02,0,RMSE,rmse,lower,0.23785345573254446,-0.23785345573254446,0.23747813004889556,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p02,0,RMSE,rmse,lower,0.23340410361335404,-0.23340410361335404,0.2382911617531396,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p02,0,RMSE,rmse,lower,0.23785345573254446,-0.23785345573254446,0.23747813004889556,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p02,0,RMSE,rmse,lower,0.23785345573254446,-0.23785345573254446,0.23747813004889556,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p02,0,RMSE,rmse,lower,0.23340410361335404,-0.23340410361335404,0.2382911617531396,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p02,0,RMSE,rmse,lower,0.23340410361335404,-0.23340410361335404,0.2382911617531396,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p005,0,RMSE,rmse,lower,0.2390328497980867,-0.2390328497980867,0.24063334636332392,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p01,0,RMSE,rmse,lower,0.23909857764435258,-0.23909857764435258,0.24513721764174634,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p005,0,RMSE,rmse,lower,0.2337388066681912,-0.2337388066681912,0.2344355028261131,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p001,0,RMSE,rmse,lower,0.2390544572113064,-0.2390544572113064,0.25549556988031324,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p01,0,RMSE,rmse,lower,0.23356265910507878,-0.23356265910507878,0.23721029814115205,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p001,0,RMSE,rmse,lower,0.23362806029940672,-0.23362806029940672,0.23621330151542805,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p01,0,RMSE,rmse,lower,0.23909857764435258,-0.23909857764435258,0.24513721764174634,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p01,0,RMSE,rmse,lower,0.23356265910507878,-0.23356265910507878,0.23721029814115205,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p001,0,RMSE,rmse,lower,0.2390544572113064,-0.2390544572113064,0.25549556988031324,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p005,0,RMSE,rmse,lower,0.2390328497980867,-0.2390328497980867,0.24063334636332392,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p001,0,RMSE,rmse,lower,0.2390544572113064,-0.2390544572113064,0.25549556988031324,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p001_l0p005,0,RMSE,rmse,lower,0.2337388066681912,-0.2337388066681912,0.2344355028261131,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p001,0,RMSE,rmse,lower,0.23362806029940672,-0.23362806029940672,0.23621330151542805,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n1em05_l0p001,0,RMSE,rmse,lower,0.23362806029940672,-0.23362806029940672,0.23621330151542805,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p001,0,RMSE,rmse,lower,0.2390544572113064,-0.2390544572113064,0.25549556988031324,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p01,0,RMSE,rmse,lower,0.23909857764435258,-0.23909857764435258,0.24513721764174634,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p005,0,RMSE,rmse,lower,0.2390328497980867,-0.2390328497980867,0.24063334636332392,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p001,0,RMSE,rmse,lower,0.23362806029940672,-0.23362806029940672,0.23621330151542805,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p005,0,RMSE,rmse,lower,0.2337388066681912,-0.2337388066681912,0.2344355028261131,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p01,0,RMSE,rmse,lower,0.23356265910507878,-0.23356265910507878,0.23721029814115205,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p01,0,RMSE,rmse,lower,0.23909857764435258,-0.23909857764435258,0.24513721764174634,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p005,0,RMSE,rmse,lower,0.2390328497980867,-0.2390328497980867,0.24063334636332392,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0_l0p01,0,RMSE,rmse,lower,0.23356265910507878,-0.23356265910507878,0.23721029814115205,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r1_n0p0001_l0p005,0,RMSE,rmse,lower,0.2337388066681912,-0.2337388066681912,0.2344355028261131,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p02,0,RMSE,rmse,lower,0.23785555444789067,-0.23785555444789067,0.23748897727684737,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p005,0,RMSE,rmse,lower,0.23907342929415865,-0.23907342929415865,0.23310698183908637,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p01,0,RMSE,rmse,lower,0.2376162677316268,-0.2376162677316268,0.23517437354931303,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p02,0,RMSE,rmse,lower,0.23340542834677785,-0.23340542834677785,0.23829137280333715,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p005,0,RMSE,rmse,lower,0.2332399859921054,-0.2332399859921054,0.2326072306387775,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p01,0,RMSE,rmse,lower,0.2335073783441898,-0.2335073783441898,0.2329070852784964,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p01,0,RMSE,rmse,lower,0.23798534123823967,-0.23798534123823967,0.2395424764805503,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p01,0,RMSE,rmse,lower,0.23320259273091312,-0.23320259273091312,0.23641549221562166,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p001,0,RMSE,rmse,lower,0.23905700508983188,-0.23905700508983188,0.2554892709638668,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p001,0,RMSE,rmse,lower,0.23363252496566134,-0.23363252496566134,0.2362188764537236,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p001,0,RMSE,rmse,lower,0.23716996532706527,-0.23716996532706527,0.23470593771148443,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p001,0,RMSE,rmse,lower,0.23379921122284117,-0.23379921122284117,0.23898482806189927,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p005,0,RMSE,rmse,lower,0.23885129531058336,-0.23885129531058336,0.24654667666950852,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p005,0,RMSE,rmse,lower,0.23374365968826152,-0.23374365968826152,0.23258765102786882,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p02,0,RMSE,rmse,lower,0.23720560242248867,-0.23720560242248867,0.23787240614757268,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n1em05_l0p02,0,RMSE,rmse,lower,0.23350963577191214,-0.23350963577191214,0.2393402580368093,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p005,0,RMSE,rmse,lower,0.23901989843968774,-0.23901989843968774,0.24063495640158664,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p005,0,RMSE,rmse,lower,0.23373660723675202,-0.23373660723675202,0.23443243594536417,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p01,0,RMSE,rmse,lower,0.23912916812165463,-0.23912916812165463,0.23873190398066874,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p01,0,RMSE,rmse,lower,0.2333667286677465,-0.2333667286677465,0.2358601642328826,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p01,0,RMSE,rmse,lower,0.239085684381371,-0.239085684381371,0.24513387433279307,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p005,0,RMSE,rmse,lower,0.23811064537966578,-0.23811064537966578,0.24336730552230365,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0_l0p01,0,RMSE,rmse,lower,0.23356035433981295,-0.23356035433981295,0.2372102431750833,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p005,0,RMSE,rmse,lower,0.23317303008072252,-0.23317303008072252,0.23484075680143124,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p001,0,RMSE,rmse,lower,0.23798803361544055,-0.23798803361544055,0.23971064008247925,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p001,0,RMSE,rmse,lower,0.23342205869781213,-0.23342205869781213,0.23945135735112927,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p02,0,RMSE,rmse,lower,0.23771204736707321,-0.23771204736707321,0.24308994802985293,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p02,0,RMSE,rmse,lower,0.2334750859643239,-0.2334750859643239,0.23518646749461922,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p02,0,RMSE,rmse,lower,0.23792787050471845,-0.23792787050471845,0.24466013756612576,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p0001_l0p02,0,RMSE,rmse,lower,0.23322026782389782,-0.23322026782389782,0.23659461735657236,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p001,0,RMSE,rmse,lower,0.24061645579340615,-0.24061645579340615,0.2630298852149786,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r2_n0p001_l0p001,0,RMSE,rmse,lower,0.23376432971584016,-0.23376432971584016,0.24301364183447563,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p02,0,RMSE,rmse,lower,0.23785555444789067,-0.23785555444789067,0.23748897727684737,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p02,0,RMSE,rmse,lower,0.23340542834677785,-0.23340542834677785,0.23829137280333715,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p01,0,RMSE,rmse,lower,0.2383569065742623,-0.2383569065742623,0.244620972160157,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p01,0,RMSE,rmse,lower,0.2334130414280393,-0.2334130414280393,0.23649611689361716,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p02,0,RMSE,rmse,lower,0.23747199640064248,-0.23747199640064248,0.24725484176674034,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p02,0,RMSE,rmse,lower,0.23366474775887086,-0.23366474775887086,0.23646780882908092,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p005,0,RMSE,rmse,lower,0.23814119850842008,-0.23814119850842008,0.2663355327951239,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p005,0,RMSE,rmse,lower,0.23343416363529057,-0.23343416363529057,0.234433762813203,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p02,0,RMSE,rmse,lower,0.23869658059774151,-0.23869658059774151,0.24052100411353006,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p02,0,RMSE,rmse,lower,0.23353358071122673,-0.23353358071122673,0.23738632838570875,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p01,0,RMSE,rmse,lower,0.239085684381371,-0.239085684381371,0.24513387433279307,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p01,0,RMSE,rmse,lower,0.23744052562658574,-0.23744052562658574,0.2382342886377602,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p01,0,RMSE,rmse,lower,0.23356035433981295,-0.23356035433981295,0.2372102431750833,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p01,0,RMSE,rmse,lower,0.23321172995482362,-0.23321172995482362,0.234184776040884,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p005,0,RMSE,rmse,lower,0.23901989843968774,-0.23901989843968774,0.24063495640158664,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p005,0,RMSE,rmse,lower,0.23373660723675202,-0.23373660723675202,0.23443243594536417,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p02,0,RMSE,rmse,lower,0.23843109390982442,-0.23843109390982442,0.24318669684901736,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p02,0,RMSE,rmse,lower,0.2334736499346226,-0.2334736499346226,0.23433651608184083,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p001,0,RMSE,rmse,lower,0.23905700508983188,-0.23905700508983188,0.2554892709638668,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0_l0p001,0,RMSE,rmse,lower,0.23363252496566134,-0.23363252496566134,0.2362188764537236,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p01,0,RMSE,rmse,lower,0.23850314174188256,-0.23850314174188256,0.23244720609452174,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p01,0,RMSE,rmse,lower,0.233651251716616,-0.233651251716616,0.2427291888102559,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p001,0,RMSE,rmse,lower,0.2388823307131178,-0.2388823307131178,0.25624351202497053,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p001,0,RMSE,rmse,lower,0.23408501487488748,-0.23408501487488748,0.23576018763280351,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p005,0,RMSE,rmse,lower,0.23844761590759667,-0.23844761590759667,0.2416540932193429,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n1em05_l0p005,0,RMSE,rmse,lower,0.23314779382394482,-0.23314779382394482,0.23448866631035667,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p001,0,RMSE,rmse,lower,0.23779726122481318,-0.23779726122481318,0.24153146027417532,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p0001_l0p001,0,RMSE,rmse,lower,0.23334639858951206,-0.23334639858951206,0.23179329091828219,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p005,0,RMSE,rmse,lower,0.239770922958097,-0.239770922958097,0.23915230117801708,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p005,0,RMSE,rmse,lower,0.23367947054972119,-0.23367947054972119,0.23861273393383206,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p001,0,RMSE,rmse,lower,0.23828438459742213,-0.23828438459742213,0.2530485190714714,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_cf_r4_n0p001_l0p001,0,RMSE,rmse,lower,0.23357213312617114,-0.23357213312617114,0.23844724876452805,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p00025,0,RMSE,rmse,lower,0.23857661998288315,-0.23857661998288315,0.2544202590204169,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p00025,0,RMSE,rmse,lower,0.23320292020753594,-0.23320292020753594,0.23031042609193297,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p00025,0,RMSE,rmse,lower,0.23857661998288315,-0.23857661998288315,0.2544202590204169,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p00025,0,RMSE,rmse,lower,0.23320292020753594,-0.23320292020753594,0.23031042609193297,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p00025,0,RMSE,rmse,lower,0.23857661998288315,-0.23857661998288315,0.2544202590204169,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p00025,0,RMSE,rmse,lower,0.23320292020753594,-0.23320292020753594,0.23031042609193297,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p0005,0,RMSE,rmse,lower,0.2390755017179478,-0.2390755017179478,0.25168892959044736,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p0005,0,RMSE,rmse,lower,0.2333487134502395,-0.2333487134502395,0.2416932153437863,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p00025,0,RMSE,rmse,lower,0.23857661998288315,-0.23857661998288315,0.2544202590204169,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p00025,0,RMSE,rmse,lower,0.23320292020753594,-0.23320292020753594,0.23031042609193297,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p0025,0,RMSE,rmse,lower,0.2377136850270048,-0.2377136850270048,0.262345673404661,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p0025,0,RMSE,rmse,lower,0.23295802298295287,-0.23295802298295287,0.23903366020873654,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p0005,0,RMSE,rmse,lower,0.2390755017179478,-0.2390755017179478,0.25168892959044736,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p0005,0,RMSE,rmse,lower,0.2333487134502395,-0.2333487134502395,0.2416932153437863,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p0025,0,RMSE,rmse,lower,0.2377136850270048,-0.2377136850270048,0.262345673404661,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p0025,0,RMSE,rmse,lower,0.23295802298295287,-0.23295802298295287,0.23903366020873654,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p001,0,RMSE,rmse,lower,0.23949031043262045,-0.23949031043262045,0.24155888192007524,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0_rho0p001,0,RMSE,rmse,lower,0.23316146280351166,-0.23316146280351166,0.23452524296935331,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p001,0,RMSE,rmse,lower,0.23949031043262045,-0.23949031043262045,0.24155888192007524,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p0005,0,RMSE,rmse,lower,0.2390755017179478,-0.2390755017179478,0.25168892959044736,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p0005,0,RMSE,rmse,lower,0.2333487134502395,-0.2333487134502395,0.2416932153437863,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n1em05_rho0p001,0,RMSE,rmse,lower,0.23316146280351166,-0.23316146280351166,0.23452524296935331,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p0025,0,RMSE,rmse,lower,0.2377136850270048,-0.2377136850270048,0.262345673404661,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p0025,0,RMSE,rmse,lower,0.23295802298295287,-0.23295802298295287,0.23903366020873654,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p0005,0,RMSE,rmse,lower,0.2390755017179478,-0.2390755017179478,0.25168892959044736,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p001,0,RMSE,rmse,lower,0.23949031043262045,-0.23949031043262045,0.24155888192007524,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p0005,0,RMSE,rmse,lower,0.2333487134502395,-0.2333487134502395,0.2416932153437863,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p0001_rho0p001,0,RMSE,rmse,lower,0.23316146280351166,-0.23316146280351166,0.23452524296935331,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p0025,0,RMSE,rmse,lower,0.2377136850270048,-0.2377136850270048,0.262345673404661,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p0025,0,RMSE,rmse,lower,0.23295802298295287,-0.23295802298295287,0.23903366020873654,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p001,0,RMSE,rmse,lower,0.23949031043262045,-0.23949031043262045,0.24155888192007524,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r1_n0p001_rho0p001,0,RMSE,rmse,lower,0.23316146280351166,-0.23316146280351166,0.23452524296935331,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p00025,0,RMSE,rmse,lower,0.23736952064793038,-0.23736952064793038,0.24275107338262392,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p001,0,RMSE,rmse,lower,0.23826622524732466,-0.23826622524732466,0.2354789879616611,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p00025,0,RMSE,rmse,lower,0.23322604208950684,-0.23322604208950684,0.23242508060584816,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p001,0,RMSE,rmse,lower,0.23311639447730767,-0.23311639447730767,0.2316950082866651,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p0005,0,RMSE,rmse,lower,0.2383891939899224,-0.2383891939899224,0.24401096715766643,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p0005,0,RMSE,rmse,lower,0.23289370528688721,-0.23289370528688721,0.2404508003490849,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p0025,0,RMSE,rmse,lower,0.23826509952531,-0.23826509952531,0.23646242093377673,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p00025,0,RMSE,rmse,lower,0.24006717570021247,-0.24006717570021247,0.24244266573296758,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-best-head-evaluation/0,False,962 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p0025,0,RMSE,rmse,lower,0.23303048406462032,-0.23303048406462032,0.23273007610764737,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p00025,0,RMSE,rmse,lower,0.23372927564914306,-0.23372927564914306,0.2350238164160606,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-greedy-heads-evaluation/0,False,962 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p0025,0,RMSE,rmse,lower,0.23800908632875115,-0.23800908632875115,0.2395995521584212,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p0025,0,RMSE,rmse,lower,0.2331158511435021,-0.2331158511435021,0.23340534056347034,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p001,0,RMSE,rmse,lower,0.23897535038319878,-0.23897535038319878,0.24383994851478993,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p001,0,RMSE,rmse,lower,0.23343394819363691,-0.23343394819363691,0.2365000863679966,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p001,0,RMSE,rmse,lower,0.23848516306571665,-0.23848516306571665,0.24223068067706838,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p001,0,RMSE,rmse,lower,0.23318228828211082,-0.23318228828211082,0.23425661912073115,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p00025,0,RMSE,rmse,lower,0.23803283687571922,-0.23803283687571922,0.273329277420384,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p00025,0,RMSE,rmse,lower,0.23289638454580067,-0.23289638454580067,0.23281509730791192,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p0005,0,RMSE,rmse,lower,0.2393372540058719,-0.2393372540058719,0.2906284957593463,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n1em05_rho0p0005,0,RMSE,rmse,lower,0.23272791516423366,-0.23272791516423366,0.23258942887652473,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p0025,0,RMSE,rmse,lower,0.2401098998840954,-0.2401098998840954,0.24179294965081416,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0_rho0p0025,0,RMSE,rmse,lower,0.23332136469753356,-0.23332136469753356,0.2364625390906921,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p00025,0,RMSE,rmse,lower,0.23704860354665752,-0.23704860354665752,0.23874426241915847,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p00025,0,RMSE,rmse,lower,0.23324589552748487,-0.23324589552748487,0.23661615616090015,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p0025,0,RMSE,rmse,lower,0.23784205343302714,-0.23784205343302714,0.23492100218498793,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p0025,0,RMSE,rmse,lower,0.23278002032353212,-0.23278002032353212,0.23227273339288673,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p0005,0,RMSE,rmse,lower,0.23835052202476453,-0.23835052202476453,0.23699550345285816,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p0005,0,RMSE,rmse,lower,0.2333328522027255,-0.2333328522027255,0.238922935773872,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p0005,0,RMSE,rmse,lower,0.24035605783658937,-0.24035605783658937,0.2779356858657707,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p0005,0,RMSE,rmse,lower,0.23367927924707155,-0.23367927924707155,0.24524323007832846,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p001,0,RMSE,rmse,lower,0.23789169956459427,-0.23789169956459427,0.23950414627424274,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p001_rho0p001,0,RMSE,rmse,lower,0.23367845823808855,-0.23367845823808855,0.24371457881808845,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p00025,0,RMSE,rmse,lower,0.23818537080734506,-0.23818537080734506,0.23894844295090592,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p00025,0,RMSE,rmse,lower,0.2337721462391934,-0.2337721462391934,0.2337674053617211,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p0005,0,RMSE,rmse,lower,0.23826995416351013,-0.23826995416351013,0.23722919786139868,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p0005,0,RMSE,rmse,lower,0.23341469329024547,-0.23341469329024547,0.23271509315390915,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p0005,0,RMSE,rmse,lower,0.23743311228743408,-0.23743311228743408,0.24918858157196577,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p0005,0,RMSE,rmse,lower,0.23308539046397073,-0.23308539046397073,0.23474214725238218,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p0025,0,RMSE,rmse,lower,0.23785759049732064,-0.23785759049732064,0.23589133259357026,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p0025,0,RMSE,rmse,lower,0.23269480219963232,-0.23269480219963232,0.23239933834285403,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p0005,0,RMSE,rmse,lower,0.2382764815807058,-0.2382764815807058,0.24121307650092602,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p0005,0,RMSE,rmse,lower,0.23269733966625106,-0.23269733966625106,0.23483414974881817,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p0005,0,RMSE,rmse,lower,0.23686825686043878,-0.23686825686043878,0.24350702848855615,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p0005,0,RMSE,rmse,lower,0.23313960483456422,-0.23313960483456422,0.23741637847101169,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p001,0,RMSE,rmse,lower,0.23779476251258524,-0.23779476251258524,0.2405214145563063,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p001,0,RMSE,rmse,lower,0.2333871447652357,-0.2333871447652357,0.2345517842823023,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p001,0,RMSE,rmse,lower,0.23860257792551076,-0.23860257792551076,0.30043019206533295,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0_rho0p001,0,RMSE,rmse,lower,0.2336154232368663,-0.2336154232368663,0.24064628828924506,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p00025,0,RMSE,rmse,lower,0.23842471915921798,-0.23842471915921798,0.34030246469362907,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p00025,0,RMSE,rmse,lower,0.23367244804564585,-0.23367244804564585,0.25122008031651116,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p0025,0,RMSE,rmse,lower,0.2380240960109698,-0.2380240960109698,0.23356247568168645,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p0025,0,RMSE,rmse,lower,0.23277917213756988,-0.23277917213756988,0.2432190859636802,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p001,0,RMSE,rmse,lower,0.23707897142181117,-0.23707897142181117,0.23966200804896912,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p0025,0,RMSE,rmse,lower,0.2393572230996531,-0.2393572230996531,0.23868415725996342,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-best-head-evaluation/0,False,740 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n1em05_rho0p001,0,RMSE,rmse,lower,0.23314020403866242,-0.23314020403866242,0.23450263045082137,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p0025,0,RMSE,rmse,lower,0.23411584672739014,-0.23411584672739014,0.23627903325948496,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-greedy-heads-evaluation/0,False,740 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p00025,0,RMSE,rmse,lower,0.23957945554469914,-0.23957945554469914,0.2496147390992142,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p00025,0,RMSE,rmse,lower,0.23353144315489308,-0.23353144315489308,0.2478765142940872,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p00025,0,RMSE,rmse,lower,0.23889975711739106,-0.23889975711739106,0.23704205803317221,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p00025,0,RMSE,rmse,lower,0.23338687341364966,-0.23338687341364966,0.23796418466778702,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p001,0,RMSE,rmse,lower,0.2387734473283714,-0.2387734473283714,0.23954579674332094,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p0001_rho0p001,0,RMSE,rmse,lower,0.23413401771618858,-0.23413401771618858,0.23885206734563352,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p0025,0,RMSE,rmse,lower,0.2389336160662929,-0.2389336160662929,0.2358168359752649,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p0025,0,RMSE,rmse,lower,0.23389711137908714,-0.23389711137908714,0.2407960992475978,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001,0,RMSE,rmse,lower,0.23897181954001903,-0.23897181954001903,0.23496733361066027,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001,0,RMSE,rmse,lower,0.2341784288692296,-0.2341784288692296,0.23454983071534766,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01,0,RMSE,rmse,lower,0.23781902006036262,-0.23781902006036262,0.2470883298795412,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01,0,RMSE,rmse,lower,0.2335463179080623,-0.2335463179080623,0.23522276143697368,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01,0,RMSE,rmse,lower,0.23756948853650636,-0.23756948853650636,0.24224213014777235,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01,0,RMSE,rmse,lower,0.23291770554949479,-0.23291770554949479,0.23458472246540202,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001,0,RMSE,rmse,lower,0.23862914188251422,-0.23862914188251422,0.23553986357063933,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001,0,RMSE,rmse,lower,0.23305490182627153,-0.23305490182627153,0.23732565171059752,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005,0,RMSE,rmse,lower,0.23918568828057105,-0.23918568828057105,0.25293341379845147,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005,0,RMSE,rmse,lower,0.2331196304833006,-0.2331196304833006,0.23280496841127477,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005,0,RMSE,rmse,lower,0.2372450969584346,-0.2372450969584346,0.23726047684878263,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.23560357747729638,-0.23560357747729638,0.2389191702722735,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.2324412121462238,-0.2324412121462238,0.23820740696857753,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005,0,RMSE,rmse,lower,0.2328726380397863,-0.2328726380397863,0.23808236498886684,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01,0,RMSE,rmse,lower,0.23656618722639788,-0.23656618722639788,0.2443526301182123,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01,0,RMSE,rmse,lower,0.23334387616340208,-0.23334387616340208,0.2380340654227449,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001,0,RMSE,rmse,lower,0.23897181954001903,-0.23897181954001903,0.23496733361066027,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005,0,RMSE,rmse,lower,0.24058462987099405,-0.24058462987099405,0.2859647081281448,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-best-head-evaluation/0,False,1184 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001,0,RMSE,rmse,lower,0.2341784288692296,-0.2341784288692296,0.23454983071534766,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005,0,RMSE,rmse,lower,0.23316436266516308,-0.23316436266516308,0.23171057167720172,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-greedy-heads-evaluation/0,False,1184 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01,0,RMSE,rmse,lower,0.23781902006036262,-0.23781902006036262,0.2470883298795412,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01,0,RMSE,rmse,lower,0.2335463179080623,-0.2335463179080623,0.23522276143697368,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01,0,RMSE,rmse,lower,0.23756948853650636,-0.23756948853650636,0.24224213014777235,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01,0,RMSE,rmse,lower,0.23291770554949479,-0.23291770554949479,0.23458472246540202,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.23560357747729638,-0.23560357747729638,0.2389191702722735,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001,0,RMSE,rmse,lower,0.23862914188251422,-0.23862914188251422,0.23553986357063933,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.2324412121462238,-0.2324412121462238,0.23820740696857753,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001,0,RMSE,rmse,lower,0.23305490182627153,-0.23305490182627153,0.23732565171059752,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005,0,RMSE,rmse,lower,0.23918568828057105,-0.23918568828057105,0.25293341379845147,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005,0,RMSE,rmse,lower,0.2331196304833006,-0.2331196304833006,0.23280496841127477,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005,0,RMSE,rmse,lower,0.24058462987099405,-0.24058462987099405,0.2859647081281448,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-best-head-evaluation/0,False,1184 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005,0,RMSE,rmse,lower,0.23316436266516308,-0.23316436266516308,0.23171057167720172,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0,False,1184 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005,0,RMSE,rmse,lower,0.2372450969584346,-0.2372450969584346,0.23726047684878263,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005,0,RMSE,rmse,lower,0.2328726380397863,-0.2328726380397863,0.23808236498886684,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.23656618722639788,-0.23656618722639788,0.2443526301182123,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.23334387616340208,-0.23334387616340208,0.2380340654227449,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001,0,RMSE,rmse,lower,0.23897181954001903,-0.23897181954001903,0.23496733361066027,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001,0,RMSE,rmse,lower,0.2341784288692296,-0.2341784288692296,0.23454983071534766,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01,0,RMSE,rmse,lower,0.23781902006036262,-0.23781902006036262,0.2470883298795412,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01,0,RMSE,rmse,lower,0.2335463179080623,-0.2335463179080623,0.23522276143697368,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01,0,RMSE,rmse,lower,0.23756948853650636,-0.23756948853650636,0.24224213014777235,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01,0,RMSE,rmse,lower,0.23291770554949479,-0.23291770554949479,0.23458472246540202,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01,0,RMSE,rmse,lower,0.23656618722639788,-0.23656618722639788,0.2443526301182123,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01,0,RMSE,rmse,lower,0.23334387616340208,-0.23334387616340208,0.2380340654227449,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001,0,RMSE,rmse,lower,0.23862914188251422,-0.23862914188251422,0.23553986357063933,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001,0,RMSE,rmse,lower,0.23560357747729638,-0.23560357747729638,0.2389191702722735,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001,0,RMSE,rmse,lower,0.23305490182627153,-0.23305490182627153,0.23732565171059752,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001,0,RMSE,rmse,lower,0.2324412121462238,-0.2324412121462238,0.23820740696857753,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005,0,RMSE,rmse,lower,0.2372450969584346,-0.2372450969584346,0.23726047684878263,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005,0,RMSE,rmse,lower,0.2328726380397863,-0.2328726380397863,0.23808236498886684,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005,0,RMSE,rmse,lower,0.23918568828057105,-0.23918568828057105,0.25293341379845147,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005,0,RMSE,rmse,lower,0.2331196304833006,-0.2331196304833006,0.23280496841127477,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005,0,RMSE,rmse,lower,0.24058462987099405,-0.24058462987099405,0.2859647081281448,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-best-head-evaluation/0,False,1184 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005,0,RMSE,rmse,lower,0.23316436266516308,-0.23316436266516308,0.23171057167720172,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-greedy-heads-evaluation/0,False,1184 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005,0,RMSE,rmse,lower,0.23790953518728727,-0.23790953518728727,0.23821708722105353,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005,0,RMSE,rmse,lower,0.23395800051802765,-0.23395800051802765,0.23212124110153423,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01,0,RMSE,rmse,lower,0.23752709943108333,-0.23752709943108333,0.23270363118336854,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01,0,RMSE,rmse,lower,0.23302229095267976,-0.23302229095267976,0.2330139696533979,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001,0,RMSE,rmse,lower,0.23774841016444426,-0.23774841016444426,0.24035398870465244,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001,0,RMSE,rmse,lower,0.23360989779537025,-0.23360989779537025,0.24433293207334258,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001,0,RMSE,rmse,lower,0.23843676541966466,-0.23843676541966466,0.2470445431022663,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001,0,RMSE,rmse,lower,0.23360278549070665,-0.23360278549070665,0.23177048418825977,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005,0,RMSE,rmse,lower,0.23613982935624828,-0.23613982935624828,0.23869431760070778,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005,0,RMSE,rmse,lower,0.23307964467970407,-0.23307964467970407,0.23920139503937368,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.23604388522704514,-0.23604388522704514,0.2413116661749965,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.23298097729336806,-0.23298097729336806,0.23673558379896892,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01,0,RMSE,rmse,lower,0.23924971586268343,-0.23924971586268343,0.23943478014622727,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01,0,RMSE,rmse,lower,0.233123281922542,-0.233123281922542,0.2324621021496061,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005,0,RMSE,rmse,lower,0.2369960693297996,-0.2369960693297996,0.2449247661219165,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005,0,RMSE,rmse,lower,0.23320698567170814,-0.23320698567170814,0.24367480279619608,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01,0,RMSE,rmse,lower,0.23723977382385666,-0.23723977382385666,0.24324646285162105,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01,0,RMSE,rmse,lower,0.23277517922062366,-0.23277517922062366,0.239397685367944,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001,0,RMSE,rmse,lower,0.2382472279758646,-0.2382472279758646,0.2416035703731357,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005,0,RMSE,rmse,lower,0.23764755063810214,-0.23764755063810214,0.24545806951373206,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001,0,RMSE,rmse,lower,0.23350649291605036,-0.23350649291605036,0.23477688348213874,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005,0,RMSE,rmse,lower,0.23351058500178656,-0.23351058500178656,0.23619933599335108,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001,0,RMSE,rmse,lower,0.23872630190199182,-0.23872630190199182,0.23512648270487807,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001,0,RMSE,rmse,lower,0.23347937804489072,-0.23347937804489072,0.24206068964408525,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005,0,RMSE,rmse,lower,0.23800428115225924,-0.23800428115225924,0.23509819987146183,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005,0,RMSE,rmse,lower,0.23277299468998114,-0.23277299468998114,0.2357753957826797,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01,0,RMSE,rmse,lower,0.2388627040189618,-0.2388627040189618,0.24858429278757524,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01,0,RMSE,rmse,lower,0.23403326393093224,-0.23403326393093224,0.23267639873508839,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01,0,RMSE,rmse,lower,0.23798911368843165,-0.23798911368843165,0.24638280846527752,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01,0,RMSE,rmse,lower,0.2329412795410917,-0.2329412795410917,0.23349850795747007,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.23993622491293543,-0.23993622491293543,0.2477713353601464,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.23356126348868944,-0.23356126348868944,0.2502618550799444,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005,0,RMSE,rmse,lower,0.2360115531890879,-0.2360115531890879,0.23473422019733492,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005,0,RMSE,rmse,lower,0.23283208982244355,-0.23283208982244355,0.23176779995051228,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.23637143045682502,-0.23637143045682502,0.2444339973181191,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.233185579288438,-0.233185579288438,0.23514442509167285,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005,0,RMSE,rmse,lower,0.23936406325960766,-0.23936406325960766,0.23718997553210766,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001,0,RMSE,rmse,lower,0.2389538449467578,-0.2389538449467578,0.24008690564440552,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005,0,RMSE,rmse,lower,0.23354782526985796,-0.23354782526985796,0.23343612653897647,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001,0,RMSE,rmse,lower,0.2336424824585913,-0.2336424824585913,0.23903173547895684,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01,0,RMSE,rmse,lower,0.238226555972743,-0.238226555972743,0.2428780524434567,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01,0,RMSE,rmse,lower,0.23345663231013183,-0.23345663231013183,0.23687685165587416,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001,0,RMSE,rmse,lower,0.23805604517030857,-0.23805604517030857,0.24432422600723777,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001,0,RMSE,rmse,lower,0.23337935525306813,-0.23337935525306813,0.2363605083000373,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005,0,RMSE,rmse,lower,0.23774960884328808,-0.23774960884328808,0.23837226942825876,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005,0,RMSE,rmse,lower,0.23309062467822936,-0.23309062467822936,0.2363639835864881,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01,0,RMSE,rmse,lower,0.2385343005293775,-0.2385343005293775,0.24325157796763505,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01,0,RMSE,rmse,lower,0.23315395335227065,-0.23315395335227065,0.2333720363863976,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01,0,RMSE,rmse,lower,0.23735456374234598,-0.23735456374234598,0.24576564896936498,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01,0,RMSE,rmse,lower,0.23287923675610403,-0.23287923675610403,0.23928534680182742,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001,0,RMSE,rmse,lower,0.23903748624498683,-0.23903748624498683,0.23879927467488832,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001,0,RMSE,rmse,lower,0.23308791568735607,-0.23308791568735607,0.23436618629413886,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005,0,RMSE,rmse,lower,0.23781616129168393,-0.23781616129168393,0.24546581719417238,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005,0,RMSE,rmse,lower,0.23341984830196463,-0.23341984830196463,0.24027958886401746,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005,0,RMSE,rmse,lower,0.238198961838987,-0.238198961838987,0.23998827851003937,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005,0,RMSE,rmse,lower,0.2329018389418362,-0.2329018389418362,0.23679443729395636,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001,0,RMSE,rmse,lower,0.23901917370395387,-0.23901917370395387,0.23709576833229662,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001,0,RMSE,rmse,lower,0.23354370990991408,-0.23354370990991408,0.23867420719836574,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001,0,RMSE,rmse,lower,0.23745188445279078,-0.23745188445279078,0.23531645200331283,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001,0,RMSE,rmse,lower,0.2330749056995843,-0.2330749056995843,0.23692981324824752,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005,0,RMSE,rmse,lower,0.23707190033649148,-0.23707190033649148,0.2397359546762736,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005,0,RMSE,rmse,lower,0.23225748836355237,-0.23225748836355237,0.2373654401585729,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01,0,RMSE,rmse,lower,0.2384660660716776,-0.2384660660716776,0.2608031155268875,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01,0,RMSE,rmse,lower,0.23344479982230373,-0.23344479982230373,0.2326324214479151,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005,0,RMSE,rmse,lower,0.23788508328903052,-0.23788508328903052,0.2505198973693071,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005,0,RMSE,rmse,lower,0.23340701642091277,-0.23340701642091277,0.24153220831875505,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001,0,RMSE,rmse,lower,0.23792789399055542,-0.23792789399055542,0.23833634568163614,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001,0,RMSE,rmse,lower,0.23351789154964117,-0.23351789154964117,0.2420072500053204,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01,0,RMSE,rmse,lower,0.2386455487030765,-0.2386455487030765,0.27701433524933666,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01,0,RMSE,rmse,lower,0.23375630575487258,-0.23375630575487258,0.23772689559358076,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.23771184363825437,-0.23771184363825437,0.2596801100779992,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.2337060760344375,-0.2337060760344375,0.23617809041406845,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01,0,RMSE,rmse,lower,0.23901159112197984,-0.23901159112197984,0.24670214607411306,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01,0,RMSE,rmse,lower,0.2334202712313093,-0.2334202712313093,0.2351349272835184,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005,0,RMSE,rmse,lower,0.23758917502158358,-0.23758917502158358,0.23986609072189577,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005,0,RMSE,rmse,lower,0.23335307969400199,-0.23335307969400199,0.23233217223360508,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01,0,RMSE,rmse,lower,0.23859870588230647,-0.23859870588230647,0.23721550415536108,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01,0,RMSE,rmse,lower,0.23377952428751153,-0.23377952428751153,0.23516218393385976,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001,0,RMSE,rmse,lower,0.23834036265829958,-0.23834036265829958,0.2355387485424496,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-best-head-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001,0,RMSE,rmse,lower,0.2328865391175455,-0.2328865391175455,0.23281656140106766,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0,False,814 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.2357278561917395,-0.2357278561917395,0.24177902137278479,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.2330191974784263,-0.2330191974784263,0.24024111265333,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005,0,RMSE,rmse,lower,0.23782636654662812,-0.23782636654662812,0.23905601555002126,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005,0,RMSE,rmse,lower,0.2332281025812143,-0.2332281025812143,0.23533248824072742,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01,0,RMSE,rmse,lower,0.2383424727161954,-0.2383424727161954,0.23929541934545473,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01,0,RMSE,rmse,lower,0.23335380606300898,-0.23335380606300898,0.23440792329336166,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005,0,RMSE,rmse,lower,0.2361938000242818,-0.2361938000242818,0.23968299921179284,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005,0,RMSE,rmse,lower,0.23281693742350693,-0.23281693742350693,0.23764734685399194,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.236445130001989,-0.236445130001989,0.2384320235467145,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.23288295595018185,-0.23288295595018185,0.2378111877355073,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005,0,RMSE,rmse,lower,0.23844788931164226,-0.23844788931164226,0.239992780080121,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005,0,RMSE,rmse,lower,0.23278091651666263,-0.23278091651666263,0.23692641701684622,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01,0,RMSE,rmse,lower,0.23860736324692997,-0.23860736324692997,0.24541511515454548,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01,0,RMSE,rmse,lower,0.23326554764868923,-0.23326554764868923,0.24138003765619687,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001,0,RMSE,rmse,lower,0.237046057654127,-0.237046057654127,0.2530936074511429,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001,0,RMSE,rmse,lower,0.23381808394805512,-0.23381808394805512,0.230425386066968,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005,0,RMSE,rmse,lower,0.2371704208371665,-0.2371704208371665,0.23451416334598613,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005,0,RMSE,rmse,lower,0.23311429304688983,-0.23311429304688983,0.23653039079394922,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01,0,RMSE,rmse,lower,0.23633714926648194,-0.23633714926648194,0.23801691214143877,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01,0,RMSE,rmse,lower,0.2329376812231261,-0.2329376812231261,0.23722725849121748,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001,0,RMSE,rmse,lower,0.2384361951499822,-0.2384361951499822,0.23890960422803306,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001,0,RMSE,rmse,lower,0.2338562469701412,-0.2338562469701412,0.23357564989983745,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001,0,RMSE,rmse,lower,0.23738497878870984,-0.23738497878870984,0.2429803744292108,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001,0,RMSE,rmse,lower,0.23371274685689286,-0.23371274685689286,0.23853136443639264,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005,0,RMSE,rmse,lower,0.23674080812018197,-0.23674080812018197,0.24022462096222602,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-best-head-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005,0,RMSE,rmse,lower,0.2333506132153353,-0.2333506132153353,0.23993873237607374,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01,0,RMSE,rmse,lower,0.24086258324668144,-0.24086258324668144,0.2704202668685561,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-best-head-evaluation/0,False,888 +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01,0,RMSE,rmse,lower,0.23354353444721934,-0.23354353444721934,0.2390892602865205,greedy-heads,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-greedy-heads-evaluation/0,False,888 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.235050071622046,-0.235050071622046,0.23184680318718037,mean,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0,False,666 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,1,RMSE,rmse,lower,0.23618618194018487,-0.23618618194018487,0.23253236276211406,mean,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1,False,1184 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,2,RMSE,rmse,lower,0.23592642069691294,-0.23592642069691294,0.2326191697888986,mean,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2,False,888 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,2,RMSE,rmse,lower,0.23592642069691294,-0.23592642069691294,0.2326191697888986,mean,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2,False,888 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.235050071622046,-0.235050071622046,0.23184680318718037,mean,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0,False,666 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,1,RMSE,rmse,lower,0.23618618194018487,-0.23618618194018487,0.23253236276211406,mean,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1,False,1184 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.23560357747729638,-0.23560357747729638,0.2389191702722735,best-head,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,2,RMSE,rmse,lower,0.23847619665119374,-0.23847619665119374,0.2542706881867437,best-head,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/2,False,888 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,1,RMSE,rmse,lower,0.24254141605050564,-0.24254141605050564,0.24204913155172003,best-head,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/1,False,1184 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.2324412121462238,-0.2324412121462238,0.23820740696857753,greedy-heads,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,2,RMSE,rmse,lower,0.23411027740292373,-0.23411027740292373,0.24496736545125247,greedy-heads,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/2,False,888 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,1,RMSE,rmse,lower,0.23397077829789442,-0.23397077829789442,0.23276530465893142,greedy-heads,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/1,False,1184 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.23560357747729638,-0.23560357747729638,0.2389191702722735,best-head,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0,False,666 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,2,RMSE,rmse,lower,0.23847619665119374,-0.23847619665119374,0.2542706881867437,best-head,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/2,False,888 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,1,RMSE,rmse,lower,0.24254141605050564,-0.24254141605050564,0.24204913155172003,best-head,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/1,False,1184 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.2324412121462238,-0.2324412121462238,0.23820740696857753,greedy-heads,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0,False,666 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,2,RMSE,rmse,lower,0.23411027740292373,-0.23411027740292373,0.24496736545125247,greedy-heads,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/2,False,888 +rescue_sberbank_combined_confirm,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,1,RMSE,rmse,lower,0.23397077829789442,-0.23397077829789442,0.23276530465893142,greedy-heads,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1.toml,exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/1,False,1184 diff --git a/tabm_integrated/paper/exp/final_integrated_summary.csv b/tabm_integrated/paper/exp/final_integrated_summary.csv index 9443108f7..c2e9ff847 100644 --- a/tabm_integrated/paper/exp/final_integrated_summary.csv +++ b/tabm_integrated/paper/exp/final_integrated_summary.csv @@ -4,7 +4,7 @@ sberbank-housing,regression,best_rla_only,rla_rank2_noise0.001,rla_rank2_noise0. sberbank-housing,regression,best_esam_only,esam_rho0.0025,esam_rho0.0025,RMSE,lower,lower,0.23381482399811443,0.23375674885076472,-5.8075147349706535e-05,-0.02484426551756285,0.23440607616465495,0.23440607616465495,0.004011724082624796,0.23440607616465495,0.004011724082624796,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,-0.00030101718081354534,-0.12858209135682216,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_esam_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_esam_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,loss sberbank-housing,regression,best_mfb_only,mfb_keep0.8,mfb_keep0.8,RMSE,lower,lower,0.23321906081613947,0.23375674885076472,0.0005376880346252555,0.23002032551732954,0.23509963234240183,0.23509963234240183,0.003956610803973734,0.23509963234240183,0.003956610803973734,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,-0.00099457335856043,-0.4248406091169,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_mfb_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_mfb_only-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,loss sberbank-housing,regression,best_cf_fisd_only,cf_fisd_only,cf_fisd_only,RMSE,lower,lower,0.23407921163352594,0.23375674885076472,-0.00032246278276121254,-0.137948009777925,0.23309306252476805,0.23309306252476805,0.0005850816922811394,0.23309306252476805,0.0005850816922811394,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,0.00101199645907335,0.4322830371398342,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_cf_fisd_only-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_cf_fisd_only-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,weak_win -sberbank-housing,regression,best_combined,mfb_cf_fisd,mfb_cf_fisd,RMSE,lower,lower,0.23329192884803412,0.23375674885076472,0.0004648200027305982,0.1988477359545025,0.23497877374993395,0.23497877374993395,0.003370630932382365,0.23497877374993395,0.003370630932382365,3,0.2341050589838414,0.002783845646101281,greedy-heads,0.2341050589838414,0.002783845646101281,-0.0008737147660925482,-0.3732148164097788,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-greedy-heads-evaluation/2,loss +sberbank-housing,regression,best_combined,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,RMSE,lower,lower,0.23887373005966525,0.24017066037702445,0.0012969303173591962,0.5400036437936467,0.24507966333691242,0.24507966333691242,0.008112050773101143,0.24507966333691242,0.008112050773101143,3,0.2502002926581795,0.013600655537586393,best-head,0.2502002926581795,0.013600655537586393,0.005120629321267084,2.0466120430413817,3 seeds,best-head,best-head,exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml;exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml,exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0;exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1;exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2,exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/0;exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/1;exp/integrated/final/sberbank-housing/baseline_plr-best-head-evaluation/2,weak_win ecom-offers,binclass,baseline_plr,baseline_plr,baseline_plr,AUROC,higher,higher,0.6214296509780997,0.6214296509780997,0.0,0.0,0.5900580480375381,0.5900580480375381,0.0009703638630149601,0.5900580480375381,0.0009703638630149601,3,0.5900580480375381,0.0009703638630149601,mean,0.5900580480375381,0.0009703638630149601,0.0,0.0,3 seeds,mean,mean,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1.toml;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2.toml,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-evaluation/2,baseline ecom-offers,binclass,best_rla_only,rla_rank2_noise0.0001,rla_rank2_noise0.0001,AUROC,higher,higher,0.6348279404183594,0.635345130387201,-0.000517189968841536,-0.08140299564841912,0.5984469268269463,0.5984469268269463,0.002797349140724218,0.5984469268269463,0.002797349140724218,3,0.5996938524587336,0.0035785432553676923,best-head,0.5996938524587336,0.0035785432553676923,-0.001246925631787299,-0.20792703254751188,3 seeds,best-head,best-head,exp/integrated/final/ecom-offers/best_rla_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_rla_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/0;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/1;exp/integrated/final/ecom-offers/best_rla_only-best-head-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-best-head-evaluation/2,loss ecom-offers,binclass,best_esam_only,esam_rho0.001,esam_rho0.001,AUROC,higher,higher,0.6366563506545114,0.6365904008411623,6.594981334917538e-05,0.0103598504253335,0.6004196058355985,0.6004196058355985,0.003493353451214683,0.6004196058355985,0.003493353451214683,3,0.6003705999706029,0.0034459818868121214,greedy-heads,0.6003705999706029,0.0034459818868121214,4.900586499556603e-05,0.008162602398912538,3 seeds,greedy-heads,greedy-heads,exp/integrated/final/ecom-offers/best_esam_only-evaluation/0.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/1.toml;exp/integrated/final/ecom-offers/best_esam_only-evaluation/2.toml,exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/best_esam_only-greedy-heads-evaluation/2,exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/0;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/1;exp/integrated/final/ecom-offers/baseline_plr-greedy-heads-evaluation/2,weak_win diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0/report.json index 27e477e2c..022c71440 100644 --- a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0/report.json +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/0/report.json @@ -11,6 +11,8 @@ "amp": true, "seed": 0, "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, "data": { "cache": true, "path": "data/sberbank-housing", @@ -25,6 +27,11 @@ "arch_type": "tabm", "k": 32, "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, "backbone": { "n_blocks": 3, "d_block": 256, @@ -38,7 +45,7 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.9, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, @@ -52,7 +59,7 @@ "n_bins": 84 }, "cf_fisd": { - "lambda": 0.1, + "lambda": 0.001, "variant": "raw", "dataset_name": "sberbank-housing", "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", @@ -71,7 +78,7 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.9, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, @@ -81,14 +88,14 @@ "mask_seed": 0 }, "mask_stats": { - "keep_rate_actual": 0.9005101919174194, - "min_features_per_member": 338.0, - "mean_features_per_member": 353.0, - "max_features_per_member": 364.0, - "min_members_per_feature": 23.0, - "mean_members_per_feature": 28.816326141357422, + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, "max_members_per_feature": 32.0, - "mean_pairwise_jaccard": 0.8189943666728325, + "mean_pairwise_jaccard": 0.9552194924522834, "anchor_fraction_configured": 0.0, "n_anchor_members": 0.0, "core_fraction_configured": 0.0, @@ -96,7 +103,7 @@ } }, "cf_fisd": { - "lambda": 0.1, + "lambda": 0.001, "variant": "raw", "teacher_names": [ "xgb", @@ -538,7 +545,9 @@ 2 ], "n_features": 392, - "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" }, "epoch_size": 74, "ncl": { @@ -631,49 +640,49 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 888, + "best_step": 666, "metrics": { "train": { - "rmse": 0.23359725177653518, - "mae": 0.12286694347858429, - "r2": 0.651382327079773, - "score": -0.23359725177653518 + "rmse": 0.24039030054595545, + "mae": 0.1237230896949768, + "r2": 0.6308118104934692, + "score": -0.24039030054595545 }, "val": { - "rmse": 0.23765498863853576, - "mae": 0.13634204864501953, - "r2": 0.6227362155914307, - "score": -0.23765498863853576 + "rmse": 0.23560357747729638, + "mae": 0.13029368221759796, + "r2": 0.6292210817337036, + "score": -0.23560357747729638 }, "test": { - "rmse": 0.2536206620016002, - "mae": 0.175298810005188, - "r2": 0.5604493618011475, - "score": -0.2536206620016002 + "rmse": 0.2389191702722735, + "mae": 0.15265432000160217, + "r2": 0.6099308729171753, + "score": -0.2389191702722735 } }, - "time": "0:01:22.331704", + "time": "0:01:13.814362", "diversity": { "train": { - "mean_centered_corr": -0.018672151491045952, + "mean_centered_corr": -0.018871087580919266, "mean_pairwise_disagreement": NaN, - "member_std": 0.0502934493124485 + "member_std": 0.04625045508146286 }, "val": { - "mean_centered_corr": -0.01943311281502247, + "mean_centered_corr": -0.019568709656596184, "mean_pairwise_disagreement": NaN, - "member_std": 0.06968922913074493 + "member_std": 0.0614706352353096 }, "test": { - "mean_centered_corr": -0.020102957263588905, + "mean_centered_corr": -0.018940167501568794, "mean_pairwise_disagreement": NaN, - "member_std": 0.09299962222576141 + "member_std": 0.08073103427886963 } }, "chunk_size": null, "eval_batch_size": 32768, "heads": [ - 25 + 4 ], - "head_selection_time": "0:00:00.015321" + "head_selection_time": "0:00:00.015454" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1/report.json index 93cd7c674..051fe52d7 100644 --- a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1/report.json +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/1/report.json @@ -11,6 +11,8 @@ "amp": true, "seed": 1, "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, "data": { "cache": true, "path": "data/sberbank-housing", @@ -25,6 +27,11 @@ "arch_type": "tabm", "k": 32, "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, "backbone": { "n_blocks": 3, "d_block": 256, @@ -38,7 +45,7 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.9, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, @@ -52,7 +59,7 @@ "n_bins": 84 }, "cf_fisd": { - "lambda": 0.1, + "lambda": 0.001, "variant": "raw", "dataset_name": "sberbank-housing", "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", @@ -71,7 +78,7 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.9, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, @@ -81,14 +88,14 @@ "mask_seed": 1 }, "mask_stats": { - "keep_rate_actual": 0.9035395383834839, - "min_features_per_member": 338.0, - "mean_features_per_member": 354.1875, - "max_features_per_member": 365.0, - "min_members_per_feature": 23.0, - "mean_members_per_feature": 28.913265228271484, + "keep_rate_actual": 0.9749681353569031, + "min_features_per_member": 376.0, + "mean_features_per_member": 382.1875, + "max_features_per_member": 388.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.1989803314209, "max_members_per_feature": 32.0, - "mean_pairwise_jaccard": 0.8235956120500022, + "mean_pairwise_jaccard": 0.9511310051056044, "anchor_fraction_configured": 0.0, "n_anchor_members": 0.0, "core_fraction_configured": 0.0, @@ -96,7 +103,7 @@ } }, "cf_fisd": { - "lambda": 0.1, + "lambda": 0.001, "variant": "raw", "teacher_names": [ "xgb", @@ -538,7 +545,9 @@ 2 ], "n_features": 392, - "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" }, "epoch_size": 74, "ncl": { @@ -631,49 +640,49 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 592, + "best_step": 1184, "metrics": { "train": { - "rmse": 0.2391015457230593, - "mae": 0.12270964682102203, - "r2": 0.6347597241401672, - "score": -0.2391015457230593 + "rmse": 0.2171563784853777, + "mae": 0.11889506876468658, + "r2": 0.6987277865409851, + "score": -0.2171563784853777 }, "val": { - "rmse": 0.2374959967300924, - "mae": 0.12878940999507904, - "r2": 0.6232408285140991, - "score": -0.2374959967300924 + "rmse": 0.24254141605050564, + "mae": 0.13972081243991852, + "r2": 0.607062816619873, + "score": -0.24254141605050564 }, "test": { - "rmse": 0.24912023719870632, - "mae": 0.166133850812912, - "r2": 0.5759104490280151, - "score": -0.24912023719870632 + "rmse": 0.24204913155172003, + "mae": 0.15291987359523773, + "r2": 0.5996437072753906, + "score": -0.24204913155172003 } }, - "time": "0:01:08.183283", + "time": "0:01:33.744311", "diversity": { "train": { - "mean_centered_corr": -0.019805658608675003, + "mean_centered_corr": -0.017405925318598747, "mean_pairwise_disagreement": NaN, - "member_std": 0.046689730137586594 + "member_std": 0.049276966601610184 }, "val": { - "mean_centered_corr": -0.02030753530561924, + "mean_centered_corr": -0.017822835594415665, "mean_pairwise_disagreement": NaN, - "member_std": 0.059032831341028214 + "member_std": 0.07913164794445038 }, "test": { - "mean_centered_corr": -0.019823072478175163, + "mean_centered_corr": -0.01794285699725151, "mean_pairwise_disagreement": NaN, - "member_std": 0.07867440581321716 + "member_std": 0.10268954932689667 } }, "chunk_size": null, "eval_batch_size": 32768, "heads": [ - 17 + 27 ], - "head_selection_time": "0:00:00.015174" + "head_selection_time": "0:00:00.015010" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2/report.json index 163a3a5e3..bc47fc492 100644 --- a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2/report.json +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-best-head-evaluation/2/report.json @@ -11,6 +11,8 @@ "amp": true, "seed": 2, "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, "data": { "cache": true, "path": "data/sberbank-housing", @@ -25,6 +27,11 @@ "arch_type": "tabm", "k": 32, "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, "backbone": { "n_blocks": 3, "d_block": 256, @@ -38,7 +45,7 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.9, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, @@ -52,7 +59,7 @@ "n_bins": 84 }, "cf_fisd": { - "lambda": 0.1, + "lambda": 0.001, "variant": "raw", "dataset_name": "sberbank-housing", "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", @@ -71,7 +78,7 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.9, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, @@ -81,14 +88,14 @@ "mask_seed": 2 }, "mask_stats": { - "keep_rate_actual": 0.9005899429321289, - "min_features_per_member": 338.0, - "mean_features_per_member": 353.03125, - "max_features_per_member": 365.0, - "min_members_per_feature": 23.0, - "mean_members_per_feature": 28.818878173828125, + "keep_rate_actual": 0.9760841727256775, + "min_features_per_member": 372.0, + "mean_features_per_member": 382.625, + "max_features_per_member": 390.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.23469352722168, "max_members_per_feature": 32.0, - "mean_pairwise_jaccard": 0.8198662012375459, + "mean_pairwise_jaccard": 0.9532826281104112, "anchor_fraction_configured": 0.0, "n_anchor_members": 0.0, "core_fraction_configured": 0.0, @@ -96,7 +103,7 @@ } }, "cf_fisd": { - "lambda": 0.1, + "lambda": 0.001, "variant": "raw", "teacher_names": [ "xgb", @@ -538,7 +545,9 @@ 2 ], "n_features": 392, - "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" }, "epoch_size": 74, "ncl": { @@ -631,49 +640,49 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 1036, + "best_step": 888, "metrics": { "train": { - "rmse": 0.2242998063053038, - "mae": 0.12896357476711273, - "r2": 0.6785808801651001, - "score": -0.2242998063053038 + "rmse": 0.2229962847735884, + "mae": 0.12252119928598404, + "r2": 0.6823059320449829, + "score": -0.2229962847735884 }, "val": { - "rmse": 0.23866731604984137, - "mae": 0.1387324184179306, - "r2": 0.6195152997970581, - "score": -0.23866731604984137 + "rmse": 0.23847619665119374, + "mae": 0.134144127368927, + "r2": 0.6201244592666626, + "score": -0.23847619665119374 }, "test": { - "rmse": 0.2357692652387237, - "mae": 0.14540770649909973, - "r2": 0.6201484203338623, - "score": -0.2357692652387237 + "rmse": 0.2542706881867437, + "mae": 0.17485685646533966, + "r2": 0.5581934452056885, + "score": -0.2542706881867437 } }, - "time": "0:01:25.772588", + "time": "0:01:19.517367", "diversity": { "train": { - "mean_centered_corr": -0.01864209584891796, + "mean_centered_corr": -0.018493393436074257, "mean_pairwise_disagreement": NaN, - "member_std": 0.05142972990870476 + "member_std": 0.04774053022265434 }, "val": { - "mean_centered_corr": -0.018798964098095894, + "mean_centered_corr": -0.018883541226387024, "mean_pairwise_disagreement": NaN, - "member_std": 0.0795377790927887 + "member_std": 0.07628075778484344 }, "test": { - "mean_centered_corr": -0.01805640198290348, + "mean_centered_corr": -0.01829403080046177, "mean_pairwise_disagreement": NaN, - "member_std": 0.09804920852184296 + "member_std": 0.09460576623678207 } }, "chunk_size": null, "eval_batch_size": 32768, "heads": [ - 30 + 24 ], - "head_selection_time": "0:00:00.014937" + "head_selection_time": "0:00:00.015325" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml index e9f43e127..aff429e28 100644 --- a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0.toml @@ -5,6 +5,8 @@ gradient_clipping_norm = 1.0 amp = true seed = 0 head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 [data] cache = true @@ -20,6 +22,11 @@ weight_decay = 0.00021333759467820313 arch_type = "tabm" k = 32 share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 [model.backbone] n_blocks = 3 @@ -34,7 +41,7 @@ d_embedding = 28 enabled = true mask_mode = "member_fixed" mask_granularity = "feature_group" -keep_rate = 0.9 +keep_rate = 0.975 training_only = false inverted_scaling = true use_soft_mask = false @@ -47,7 +54,7 @@ mask_seed = 0 n_bins = 84 [cf_fisd] -lambda = 0.1 +lambda = 0.001 variant = "raw" dataset_name = "sberbank-housing" teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0/report.json index 78c9ce2d6..193998f2e 100644 --- a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0/report.json +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/0/report.json @@ -11,6 +11,8 @@ "amp": true, "seed": 0, "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, "data": { "cache": true, "path": "data/sberbank-housing", @@ -25,6 +27,11 @@ "arch_type": "tabm", "k": 32, "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, "backbone": { "n_blocks": 3, "d_block": 256, @@ -38,7 +45,7 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.9, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, @@ -52,7 +59,7 @@ "n_bins": 84 }, "cf_fisd": { - "lambda": 0.1, + "lambda": 0.001, "variant": "raw", "dataset_name": "sberbank-housing", "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", @@ -71,7 +78,7 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.9, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, @@ -81,14 +88,14 @@ "mask_seed": 0 }, "mask_stats": { - "keep_rate_actual": 0.9005101919174194, - "min_features_per_member": 338.0, - "mean_features_per_member": 353.0, - "max_features_per_member": 364.0, - "min_members_per_feature": 23.0, - "mean_members_per_feature": 28.816326141357422, + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, "max_members_per_feature": 32.0, - "mean_pairwise_jaccard": 0.8189943666728325, + "mean_pairwise_jaccard": 0.9552194924522834, "anchor_fraction_configured": 0.0, "n_anchor_members": 0.0, "core_fraction_configured": 0.0, @@ -96,7 +103,7 @@ } }, "cf_fisd": { - "lambda": 0.1, + "lambda": 0.001, "variant": "raw", "teacher_names": [ "xgb", @@ -538,7 +545,9 @@ 2 ], "n_features": 392, - "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" }, "epoch_size": 74, "ncl": { @@ -631,43 +640,43 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 888, + "best_step": 666, "metrics": { "train": { - "rmse": 0.22208876822485193, - "mae": 0.11730500310659409, - "r2": 0.6848864555358887, - "score": -0.22208876822485193 + "rmse": 0.23278188472143527, + "mae": 0.12037675827741623, + "r2": 0.6538118124008179, + "score": -0.23278188472143527 }, "val": { - "rmse": 0.23508488120210985, - "mae": 0.12783657014369965, - "r2": 0.6308518648147583, - "score": -0.23508488120210985 + "rmse": 0.235050071622046, + "mae": 0.12535566091537476, + "r2": 0.6309611797332764, + "score": -0.235050071622046 }, "test": { - "rmse": 0.23568789402876444, - "mae": 0.14989204704761505, - "r2": 0.6204105615615845, - "score": -0.23568789402876444 + "rmse": 0.23184680318718037, + "mae": 0.14313000440597534, + "r2": 0.6326823234558105, + "score": -0.23184680318718037 } }, - "time": "0:01:22.331704", + "time": "0:01:13.814362", "diversity": { "train": { - "mean_centered_corr": -0.018672151491045952, + "mean_centered_corr": -0.018871087580919266, "mean_pairwise_disagreement": NaN, - "member_std": 0.0502934493124485 + "member_std": 0.04625045508146286 }, "val": { - "mean_centered_corr": -0.01943311281502247, + "mean_centered_corr": -0.019568709656596184, "mean_pairwise_disagreement": NaN, - "member_std": 0.06968922913074493 + "member_std": 0.0614706352353096 }, "test": { - "mean_centered_corr": -0.020102957263588905, + "mean_centered_corr": -0.018940167501568794, "mean_pairwise_disagreement": NaN, - "member_std": 0.09299962222576141 + "member_std": 0.08073103427886963 } }, "chunk_size": null, diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml index c4ce180b9..ad01c71b3 100644 --- a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1.toml @@ -5,6 +5,8 @@ gradient_clipping_norm = 1.0 amp = true seed = 1 head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 [data] cache = true @@ -20,6 +22,11 @@ weight_decay = 0.00021333759467820313 arch_type = "tabm" k = 32 share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 [model.backbone] n_blocks = 3 @@ -34,7 +41,7 @@ d_embedding = 28 enabled = true mask_mode = "member_fixed" mask_granularity = "feature_group" -keep_rate = 0.9 +keep_rate = 0.975 training_only = false inverted_scaling = true use_soft_mask = false @@ -47,7 +54,7 @@ mask_seed = 1 n_bins = 84 [cf_fisd] -lambda = 0.1 +lambda = 0.001 variant = "raw" dataset_name = "sberbank-housing" teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1/report.json index b5d41c5d0..f4c5d18c3 100644 --- a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1/report.json +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/1/report.json @@ -11,6 +11,8 @@ "amp": true, "seed": 1, "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, "data": { "cache": true, "path": "data/sberbank-housing", @@ -25,6 +27,11 @@ "arch_type": "tabm", "k": 32, "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, "backbone": { "n_blocks": 3, "d_block": 256, @@ -38,7 +45,7 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.9, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, @@ -52,7 +59,7 @@ "n_bins": 84 }, "cf_fisd": { - "lambda": 0.1, + "lambda": 0.001, "variant": "raw", "dataset_name": "sberbank-housing", "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", @@ -71,7 +78,7 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.9, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, @@ -81,14 +88,14 @@ "mask_seed": 1 }, "mask_stats": { - "keep_rate_actual": 0.9035395383834839, - "min_features_per_member": 338.0, - "mean_features_per_member": 354.1875, - "max_features_per_member": 365.0, - "min_members_per_feature": 23.0, - "mean_members_per_feature": 28.913265228271484, + "keep_rate_actual": 0.9749681353569031, + "min_features_per_member": 376.0, + "mean_features_per_member": 382.1875, + "max_features_per_member": 388.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.1989803314209, "max_members_per_feature": 32.0, - "mean_pairwise_jaccard": 0.8235956120500022, + "mean_pairwise_jaccard": 0.9511310051056044, "anchor_fraction_configured": 0.0, "n_anchor_members": 0.0, "core_fraction_configured": 0.0, @@ -96,7 +103,7 @@ } }, "cf_fisd": { - "lambda": 0.1, + "lambda": 0.001, "variant": "raw", "teacher_names": [ "xgb", @@ -538,7 +545,9 @@ 2 ], "n_features": 392, - "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" }, "epoch_size": 74, "ncl": { @@ -631,43 +640,43 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 592, + "best_step": 1184, "metrics": { "train": { - "rmse": 0.2377225313298363, - "mae": 0.12293801456689835, - "r2": 0.638960599899292, - "score": -0.2377225313298363 + "rmse": 0.20665772048240127, + "mae": 0.11000452935695648, + "r2": 0.7271542549133301, + "score": -0.20665772048240127 }, "val": { - "rmse": 0.23530641494725335, - "mae": 0.12682271003723145, - "r2": 0.6301558017730713, - "score": -0.23530641494725335 + "rmse": 0.23618618194018487, + "mae": 0.12565438449382782, + "r2": 0.6273850798606873, + "score": -0.23618618194018487 }, "test": { - "rmse": 0.23169352906704774, - "mae": 0.14042755961418152, - "r2": 0.6331678628921509, - "score": -0.23169352906704774 + "rmse": 0.23253236276211406, + "mae": 0.1399872750043869, + "r2": 0.6305068731307983, + "score": -0.23253236276211406 } }, - "time": "0:01:08.183283", + "time": "0:01:33.744311", "diversity": { "train": { - "mean_centered_corr": -0.019805658608675003, + "mean_centered_corr": -0.017405925318598747, "mean_pairwise_disagreement": NaN, - "member_std": 0.046689730137586594 + "member_std": 0.049276966601610184 }, "val": { - "mean_centered_corr": -0.02030753530561924, + "mean_centered_corr": -0.017822835594415665, "mean_pairwise_disagreement": NaN, - "member_std": 0.059032831341028214 + "member_std": 0.07913164794445038 }, "test": { - "mean_centered_corr": -0.019823072478175163, + "mean_centered_corr": -0.01794285699725151, "mean_pairwise_disagreement": NaN, - "member_std": 0.07867440581321716 + "member_std": 0.10268954932689667 } }, "chunk_size": null, diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml index 4f57370d4..86addbb91 100644 --- a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2.toml @@ -5,6 +5,8 @@ gradient_clipping_norm = 1.0 amp = true seed = 2 head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 [data] cache = true @@ -20,6 +22,11 @@ weight_decay = 0.00021333759467820313 arch_type = "tabm" k = 32 share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 [model.backbone] n_blocks = 3 @@ -34,7 +41,7 @@ d_embedding = 28 enabled = true mask_mode = "member_fixed" mask_granularity = "feature_group" -keep_rate = 0.9 +keep_rate = 0.975 training_only = false inverted_scaling = true use_soft_mask = false @@ -47,7 +54,7 @@ mask_seed = 2 n_bins = 84 [cf_fisd] -lambda = 0.1 +lambda = 0.001 variant = "raw" dataset_name = "sberbank-housing" teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2/report.json index d09b3432b..65d6d6497 100644 --- a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2/report.json +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-evaluation/2/report.json @@ -11,6 +11,8 @@ "amp": true, "seed": 2, "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, "data": { "cache": true, "path": "data/sberbank-housing", @@ -25,6 +27,11 @@ "arch_type": "tabm", "k": 32, "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, "backbone": { "n_blocks": 3, "d_block": 256, @@ -38,7 +45,7 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.9, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, @@ -52,7 +59,7 @@ "n_bins": 84 }, "cf_fisd": { - "lambda": 0.1, + "lambda": 0.001, "variant": "raw", "dataset_name": "sberbank-housing", "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", @@ -71,7 +78,7 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.9, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, @@ -81,14 +88,14 @@ "mask_seed": 2 }, "mask_stats": { - "keep_rate_actual": 0.9005899429321289, - "min_features_per_member": 338.0, - "mean_features_per_member": 353.03125, - "max_features_per_member": 365.0, - "min_members_per_feature": 23.0, - "mean_members_per_feature": 28.818878173828125, + "keep_rate_actual": 0.9760841727256775, + "min_features_per_member": 372.0, + "mean_features_per_member": 382.625, + "max_features_per_member": 390.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.23469352722168, "max_members_per_feature": 32.0, - "mean_pairwise_jaccard": 0.8198662012375459, + "mean_pairwise_jaccard": 0.9532826281104112, "anchor_fraction_configured": 0.0, "n_anchor_members": 0.0, "core_fraction_configured": 0.0, @@ -96,7 +103,7 @@ } }, "cf_fisd": { - "lambda": 0.1, + "lambda": 0.001, "variant": "raw", "teacher_names": [ "xgb", @@ -538,7 +545,9 @@ 2 ], "n_features": 392, - "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" }, "epoch_size": 74, "ncl": { @@ -631,43 +640,43 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 1036, + "best_step": 888, "metrics": { "train": { - "rmse": 0.21596792187571293, - "mae": 0.11686611920595169, - "r2": 0.7020163536071777, - "score": -0.21596792187571293 + "rmse": 0.21767157152228817, + "mae": 0.11745961010456085, + "r2": 0.6972966194152832, + "score": -0.21767157152228817 }, "val": { - "rmse": 0.2356153727014129, - "mae": 0.1266695261001587, - "r2": 0.6291840076446533, - "score": -0.2356153727014129 + "rmse": 0.23592642069691294, + "mae": 0.12567678093910217, + "r2": 0.6282042264938354, + "score": -0.23592642069691294 }, "test": { - "rmse": 0.2360324823167294, - "mae": 0.15042589604854584, - "r2": 0.6192997694015503, - "score": -0.2360324823167294 + "rmse": 0.2326191697888986, + "mae": 0.14303357899188995, + "r2": 0.6302309036254883, + "score": -0.2326191697888986 } }, - "time": "0:01:25.772588", + "time": "0:01:19.517367", "diversity": { "train": { - "mean_centered_corr": -0.01864209584891796, + "mean_centered_corr": -0.018493393436074257, "mean_pairwise_disagreement": NaN, - "member_std": 0.05142972990870476 + "member_std": 0.04774053022265434 }, "val": { - "mean_centered_corr": -0.018798964098095894, + "mean_centered_corr": -0.018883541226387024, "mean_pairwise_disagreement": NaN, - "member_std": 0.0795377790927887 + "member_std": 0.07628075778484344 }, "test": { - "mean_centered_corr": -0.01805640198290348, + "mean_centered_corr": -0.01829403080046177, "mean_pairwise_disagreement": NaN, - "member_std": 0.09804920852184296 + "member_std": 0.09460576623678207 } }, "chunk_size": null, diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0/report.json index 635150edd..55f69c783 100644 --- a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0/report.json +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/0/report.json @@ -11,6 +11,8 @@ "amp": true, "seed": 0, "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, "data": { "cache": true, "path": "data/sberbank-housing", @@ -25,6 +27,11 @@ "arch_type": "tabm", "k": 32, "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, "backbone": { "n_blocks": 3, "d_block": 256, @@ -38,7 +45,7 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.9, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, @@ -52,7 +59,7 @@ "n_bins": 84 }, "cf_fisd": { - "lambda": 0.1, + "lambda": 0.001, "variant": "raw", "dataset_name": "sberbank-housing", "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", @@ -71,7 +78,7 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.9, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, @@ -81,14 +88,14 @@ "mask_seed": 0 }, "mask_stats": { - "keep_rate_actual": 0.9005101919174194, - "min_features_per_member": 338.0, - "mean_features_per_member": 353.0, - "max_features_per_member": 364.0, - "min_members_per_feature": 23.0, - "mean_members_per_feature": 28.816326141357422, + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, "max_members_per_feature": 32.0, - "mean_pairwise_jaccard": 0.8189943666728325, + "mean_pairwise_jaccard": 0.9552194924522834, "anchor_fraction_configured": 0.0, "n_anchor_members": 0.0, "core_fraction_configured": 0.0, @@ -96,7 +103,7 @@ } }, "cf_fisd": { - "lambda": 0.1, + "lambda": 0.001, "variant": "raw", "teacher_names": [ "xgb", @@ -538,7 +545,9 @@ 2 ], "n_features": 392, - "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" }, "epoch_size": 74, "ncl": { @@ -631,58 +640,53 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 888, + "best_step": 666, "metrics": { "train": { - "rmse": 0.22343022528056483, - "mae": 0.11760932952165604, - "r2": 0.6810683012008667, - "score": -0.22343022528056483 + "rmse": 0.23573796221874224, + "mae": 0.12117554992437363, + "r2": 0.6449635028839111, + "score": -0.23573796221874224 }, "val": { - "rmse": 0.23320900639668657, - "mae": 0.12497496604919434, - "r2": 0.6367197036743164, - "score": -0.23320900639668657 + "rmse": 0.2324412121462238, + "mae": 0.12522703409194946, + "r2": 0.6391078233718872, + "score": -0.2324412121462238 }, "test": { - "rmse": 0.23362227205150313, - "mae": 0.14383481442928314, - "r2": 0.6270350217819214, - "score": -0.23362227205150313 + "rmse": 0.23820740696857753, + "mae": 0.1556985229253769, + "r2": 0.6122515201568604, + "score": -0.23820740696857753 } }, - "time": "0:01:22.331704", + "time": "0:01:13.814362", "diversity": { "train": { - "mean_centered_corr": -0.018672151491045952, + "mean_centered_corr": -0.018871087580919266, "mean_pairwise_disagreement": NaN, - "member_std": 0.0502934493124485 + "member_std": 0.04625045508146286 }, "val": { - "mean_centered_corr": -0.01943311281502247, + "mean_centered_corr": -0.019568709656596184, "mean_pairwise_disagreement": NaN, - "member_std": 0.06968922913074493 + "member_std": 0.0614706352353096 }, "test": { - "mean_centered_corr": -0.020102957263588905, + "mean_centered_corr": -0.018940167501568794, "mean_pairwise_disagreement": NaN, - "member_std": 0.09299962222576141 + "member_std": 0.08073103427886963 } }, "chunk_size": null, "eval_batch_size": 32768, "heads": [ - 25, - 20, - 29, - 19, - 0, - 12, 4, - 20, - 21, - 19 + 0, + 1, + 25, + 21 ], - "head_selection_time": "0:00:00.159678" + "head_selection_time": "0:00:00.088240" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1/report.json index 3117264d5..9dcd68238 100644 --- a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1/report.json +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/1/report.json @@ -11,6 +11,8 @@ "amp": true, "seed": 1, "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, "data": { "cache": true, "path": "data/sberbank-housing", @@ -25,6 +27,11 @@ "arch_type": "tabm", "k": 32, "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, "backbone": { "n_blocks": 3, "d_block": 256, @@ -38,7 +45,7 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.9, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, @@ -52,7 +59,7 @@ "n_bins": 84 }, "cf_fisd": { - "lambda": 0.1, + "lambda": 0.001, "variant": "raw", "dataset_name": "sberbank-housing", "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", @@ -71,7 +78,7 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.9, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, @@ -81,14 +88,14 @@ "mask_seed": 1 }, "mask_stats": { - "keep_rate_actual": 0.9035395383834839, - "min_features_per_member": 338.0, - "mean_features_per_member": 354.1875, - "max_features_per_member": 365.0, - "min_members_per_feature": 23.0, - "mean_members_per_feature": 28.913265228271484, + "keep_rate_actual": 0.9749681353569031, + "min_features_per_member": 376.0, + "mean_features_per_member": 382.1875, + "max_features_per_member": 388.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.1989803314209, "max_members_per_feature": 32.0, - "mean_pairwise_jaccard": 0.8235956120500022, + "mean_pairwise_jaccard": 0.9511310051056044, "anchor_fraction_configured": 0.0, "n_anchor_members": 0.0, "core_fraction_configured": 0.0, @@ -96,7 +103,7 @@ } }, "cf_fisd": { - "lambda": 0.1, + "lambda": 0.001, "variant": "raw", "teacher_names": [ "xgb", @@ -538,7 +545,9 @@ 2 ], "n_features": 392, - "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" }, "epoch_size": 74, "ncl": { @@ -631,53 +640,64 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 592, + "best_step": 1184, "metrics": { "train": { - "rmse": 0.2359836526912841, - "mae": 0.12225823104381561, - "r2": 0.6442230939865112, - "score": -0.2359836526912841 + "rmse": 0.20580531574085753, + "mae": 0.11071916669607162, + "r2": 0.7294004559516907, + "score": -0.20580531574085753 }, "val": { - "rmse": 0.2332021853813015, - "mae": 0.12649986147880554, - "r2": 0.6367409229278564, - "score": -0.2332021853813015 + "rmse": 0.23397077829789442, + "mae": 0.12587550282478333, + "r2": 0.6343424916267395, + "score": -0.23397077829789442 }, "test": { - "rmse": 0.2324977398321572, - "mae": 0.1440209597349167, - "r2": 0.6306169033050537, - "score": -0.2324977398321572 + "rmse": 0.23276530465893142, + "mae": 0.13940665125846863, + "r2": 0.6297661662101746, + "score": -0.23276530465893142 } }, - "time": "0:01:08.183283", + "time": "0:01:33.744311", "diversity": { "train": { - "mean_centered_corr": -0.019805658608675003, + "mean_centered_corr": -0.017405925318598747, "mean_pairwise_disagreement": NaN, - "member_std": 0.046689730137586594 + "member_std": 0.049276966601610184 }, "val": { - "mean_centered_corr": -0.02030753530561924, + "mean_centered_corr": -0.017822835594415665, "mean_pairwise_disagreement": NaN, - "member_std": 0.059032831341028214 + "member_std": 0.07913164794445038 }, "test": { - "mean_centered_corr": -0.019823072478175163, + "mean_centered_corr": -0.01794285699725151, "mean_pairwise_disagreement": NaN, - "member_std": 0.07867440581321716 + "member_std": 0.10268954932689667 } }, "chunk_size": null, "eval_batch_size": 32768, "heads": [ - 17, 27, - 2, + 6, + 1, 15, - 28 + 2, + 13, + 0, + 6, + 1, + 29, + 31, + 5, + 1, + 21, + 6, + 22 ], - "head_selection_time": "0:00:00.087511" + "head_selection_time": "0:00:00.247440" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2/report.json index d4ad57cf2..88ac1a9ce 100644 --- a/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2/report.json +++ b/tabm_integrated/paper/exp/integrated/final/sberbank-housing/best_combined-greedy-heads-evaluation/2/report.json @@ -11,6 +11,8 @@ "amp": true, "seed": 2, "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, "data": { "cache": true, "path": "data/sberbank-housing", @@ -25,6 +27,11 @@ "arch_type": "tabm", "k": 32, "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, "backbone": { "n_blocks": 3, "d_block": 256, @@ -38,7 +45,7 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.9, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, @@ -52,7 +59,7 @@ "n_bins": 84 }, "cf_fisd": { - "lambda": 0.1, + "lambda": 0.001, "variant": "raw", "dataset_name": "sberbank-housing", "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", @@ -71,7 +78,7 @@ "enabled": true, "mask_mode": "member_fixed", "mask_granularity": "feature_group", - "keep_rate": 0.9, + "keep_rate": 0.975, "training_only": false, "inverted_scaling": true, "use_soft_mask": false, @@ -81,14 +88,14 @@ "mask_seed": 2 }, "mask_stats": { - "keep_rate_actual": 0.9005899429321289, - "min_features_per_member": 338.0, - "mean_features_per_member": 353.03125, - "max_features_per_member": 365.0, - "min_members_per_feature": 23.0, - "mean_members_per_feature": 28.818878173828125, + "keep_rate_actual": 0.9760841727256775, + "min_features_per_member": 372.0, + "mean_features_per_member": 382.625, + "max_features_per_member": 390.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.23469352722168, "max_members_per_feature": 32.0, - "mean_pairwise_jaccard": 0.8198662012375459, + "mean_pairwise_jaccard": 0.9532826281104112, "anchor_fraction_configured": 0.0, "n_anchor_members": 0.0, "core_fraction_configured": 0.0, @@ -96,7 +103,7 @@ } }, "cf_fisd": { - "lambda": 0.1, + "lambda": 0.001, "variant": "raw", "teacher_names": [ "xgb", @@ -538,7 +545,9 @@ 2 ], "n_features": 392, - "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" }, "epoch_size": 74, "ncl": { @@ -631,57 +640,57 @@ "extra_paths_freeze_fraction": 0.0, "extra_paths_freeze_until_epoch": 0 }, - "best_step": 1036, + "best_step": 888, "metrics": { "train": { - "rmse": 0.21572123708815089, - "mae": 0.11632093042135239, - "r2": 0.702696681022644, - "score": -0.21572123708815089 + "rmse": 0.2210309033660134, + "mae": 0.12036030739545822, + "r2": 0.6878812313079834, + "score": -0.2210309033660134 }, "val": { - "rmse": 0.23346459476611428, - "mae": 0.12624695897102356, - "r2": 0.635922908782959, - "score": -0.23346459476611428 + "rmse": 0.23411027740292373, + "mae": 0.128285214304924, + "r2": 0.633906364440918, + "score": -0.23411027740292373 }, "test": { - "rmse": 0.23881630936614148, - "mae": 0.15446454286575317, - "r2": 0.6102666854858398, - "score": -0.23881630936614148 + "rmse": 0.24496736545125247, + "mae": 0.16705775260925293, + "r2": 0.589931845664978, + "score": -0.24496736545125247 } }, - "time": "0:01:25.772588", + "time": "0:01:19.517367", "diversity": { "train": { - "mean_centered_corr": -0.01864209584891796, + "mean_centered_corr": -0.018493393436074257, "mean_pairwise_disagreement": NaN, - "member_std": 0.05142972990870476 + "member_std": 0.04774053022265434 }, "val": { - "mean_centered_corr": -0.018798964098095894, + "mean_centered_corr": -0.018883541226387024, "mean_pairwise_disagreement": NaN, - "member_std": 0.0795377790927887 + "member_std": 0.07628075778484344 }, "test": { - "mean_centered_corr": -0.01805640198290348, + "mean_centered_corr": -0.01829403080046177, "mean_pairwise_disagreement": NaN, - "member_std": 0.09804920852184296 + "member_std": 0.09460576623678207 } }, "chunk_size": null, "eval_batch_size": 32768, "heads": [ - 30, - 2, 24, - 19, - 20, - 9, - 15, + 4, + 27, + 6, + 3, + 30, + 5, 27, - 2 + 4 ], - "head_selection_time": "0:00:00.143383" + "head_selection_time": "0:00:00.145674" } \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/manifest_sberbank_combined_rescue_final.txt b/tabm_integrated/paper/exp/integrated/manifest_sberbank_combined_rescue_final.txt new file mode 100644 index 000000000..7a579dfa0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/manifest_sberbank_combined_rescue_final.txt @@ -0,0 +1,6 @@ +exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1.toml +exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2.toml +exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1.toml +exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2.toml diff --git a/tabm_integrated/paper/exp/integrated/manifest_sberbank_combined_rescue_seed0.txt b/tabm_integrated/paper/exp/integrated/manifest_sberbank_combined_rescue_seed0.txt new file mode 100644 index 000000000..59f65e1a3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/manifest_sberbank_combined_rescue_seed0.txt @@ -0,0 +1,177 @@ +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0.toml +exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0.toml diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..26a79c999 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2282046786382406, + "mae": 0.12737023830413818, + "r2": 0.6672922372817993, + "score": -0.2282046786382406 + }, + "val": { + "rmse": 0.2390544572113064, + "mae": 0.14095881581306458, + "r2": 0.6182799935340881, + "score": -0.2390544572113064 + }, + "test": { + "rmse": 0.25549556988031324, + "mae": 0.17997059226036072, + "r2": 0.5539265871047974, + "score": -0.25549556988031324 + } + }, + "time": "0:01:08.154288", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.016868" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0.toml new file mode 100644 index 000000000..3961a64e2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..d6c1b1c10 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2221833946825238, + "mae": 0.11801641434431076, + "r2": 0.6846178770065308, + "score": -0.2221833946825238 + }, + "val": { + "rmse": 0.23516710262665957, + "mae": 0.12856413424015045, + "r2": 0.6305935978889465, + "score": -0.23516710262665957 + }, + "test": { + "rmse": 0.2392893867678542, + "mae": 0.15676388144493103, + "r2": 0.6087210774421692, + "score": -0.2392893867678542 + } + }, + "time": "0:01:08.154288", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8aad1ae80 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22488731105453647, + "mae": 0.1187836155295372, + "r2": 0.6768949031829834, + "score": -0.22488731105453647 + }, + "val": { + "rmse": 0.23362806029940672, + "mae": 0.1260843127965927, + "r2": 0.6354129314422607, + "score": -0.23362806029940672 + }, + "test": { + "rmse": 0.23621330151542805, + "mae": 0.15148833394050598, + "r2": 0.6187162399291992, + "score": -0.23621330151542805 + } + }, + "time": "0:01:08.154288", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 8, + 22, + 7, + 29, + 25, + 21 + ], + "head_selection_time": "0:00:00.120526" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a0e3ff8d5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23007499064462367, + "mae": 0.12374904006719589, + "r2": 0.6618162393569946, + "score": -0.23007499064462367 + }, + "val": { + "rmse": 0.2390328497980867, + "mae": 0.13070335984230042, + "r2": 0.6183489561080933, + "score": -0.2390328497980867 + }, + "test": { + "rmse": 0.24063334636332392, + "mae": 0.15138967335224152, + "r2": 0.6043134927749634, + "score": -0.24063334636332392 + } + }, + "time": "0:01:08.381106", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015250" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0.toml new file mode 100644 index 000000000..33808ab55 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..98b430c43 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22270659663864684, + "mae": 0.11821237951517105, + "r2": 0.6831307411193848, + "score": -0.22270659663864684 + }, + "val": { + "rmse": 0.23541557264636437, + "mae": 0.1285467892885208, + "r2": 0.6298125982284546, + "score": -0.23541557264636437 + }, + "test": { + "rmse": 0.23955821427368978, + "mae": 0.15599055588245392, + "r2": 0.6078413724899292, + "score": -0.23955821427368978 + } + }, + "time": "0:01:08.381106", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..738f85078 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,657 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2256070659700792, + "mae": 0.11890602111816406, + "r2": 0.6748234033584595, + "score": -0.2256070659700792 + }, + "val": { + "rmse": 0.2337388066681912, + "mae": 0.12727881968021393, + "r2": 0.6350672245025635, + "score": -0.2337388066681912 + }, + "test": { + "rmse": 0.2344355028261131, + "mae": 0.14499156177043915, + "r2": 0.6244339346885681, + "score": -0.2344355028261131 + } + }, + "time": "0:01:08.381106", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 29, + 8, + 0, + 19, + 22 + ], + "head_selection_time": "0:00:00.102535" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..21a562273 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2301618504668323, + "mae": 0.12545494735240936, + "r2": 0.6615608930587769, + "score": -0.2301618504668323 + }, + "val": { + "rmse": 0.23909857764435258, + "mae": 0.13463132083415985, + "r2": 0.6181390285491943, + "score": -0.23909857764435258 + }, + "test": { + "rmse": 0.24513721764174634, + "mae": 0.16250674426555634, + "r2": 0.5893629789352417, + "score": -0.24513721764174634 + } + }, + "time": "0:01:09.433151", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.016053" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0.toml new file mode 100644 index 000000000..d859343b1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..e5c7ea752 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2241623618168725, + "mae": 0.11926797032356262, + "r2": 0.6789746880531311, + "score": -0.2241623618168725 + }, + "val": { + "rmse": 0.23516078988949235, + "mae": 0.12952424585819244, + "r2": 0.6306134462356567, + "score": -0.23516078988949235 + }, + "test": { + "rmse": 0.24165682950771084, + "mae": 0.16096629202365875, + "r2": 0.600940465927124, + "score": -0.24165682950771084 + } + }, + "time": "0:01:09.433151", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..aa499482f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,661 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22445249469109585, + "mae": 0.11813493072986603, + "r2": 0.6781431436538696, + "score": -0.22445249469109585 + }, + "val": { + "rmse": 0.23356265910507878, + "mae": 0.1266428679227829, + "r2": 0.6356170177459717, + "score": -0.23356265910507878 + }, + "test": { + "rmse": 0.23721029814115205, + "mae": 0.1523510068655014, + "r2": 0.6154908537864685, + "score": -0.23721029814115205 + } + }, + "time": "0:01:09.433151", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 7, + 4, + 22, + 19, + 8, + 18, + 29, + 7, + 25 + ], + "head_selection_time": "0:00:00.165917" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..da5a2ec8a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23893091875282987, + "mae": 0.12647393345832825, + "r2": 0.6352808475494385, + "score": -0.23893091875282987 + }, + "val": { + "rmse": 0.23785345573254446, + "mae": 0.13102787733078003, + "r2": 0.6221058368682861, + "score": -0.23785345573254446 + }, + "test": { + "rmse": 0.23747813004889556, + "mae": 0.14464227855205536, + "r2": 0.6146221160888672, + "score": -0.23747813004889556 + } + }, + "time": "0:01:03.669566", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.015246" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0.toml new file mode 100644 index 000000000..931d62862 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..9292482c6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23258982929101366, + "mae": 0.1219717338681221, + "r2": 0.6543828248977661, + "score": -0.23258982929101366 + }, + "val": { + "rmse": 0.23537261362228468, + "mae": 0.12668848037719727, + "r2": 0.6299476623535156, + "score": -0.23537261362228468 + }, + "test": { + "rmse": 0.23578993148696667, + "mae": 0.14991270005702972, + "r2": 0.6200817823410034, + "score": -0.23578993148696667 + } + }, + "time": "0:01:03.669566", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2cdf220da --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,661 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2347629753255239, + "mae": 0.12291525304317474, + "r2": 0.6478942632675171, + "score": -0.2347629753255239 + }, + "val": { + "rmse": 0.23340410361335404, + "mae": 0.12670208513736725, + "r2": 0.6361116170883179, + "score": -0.23340410361335404 + }, + "test": { + "rmse": 0.2382911617531396, + "mae": 0.15507616102695465, + "r2": 0.6119787693023682, + "score": -0.2382911617531396 + } + }, + "time": "0:01:03.669566", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29, + 25, + 28, + 0, + 16, + 21, + 4, + 25, + 1, + 6 + ], + "head_selection_time": "0:00:00.159810" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..dd025feaf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2282046786382406, + "mae": 0.12737023830413818, + "r2": 0.6672922372817993, + "score": -0.2282046786382406 + }, + "val": { + "rmse": 0.2390544572113064, + "mae": 0.14095881581306458, + "r2": 0.6182799935340881, + "score": -0.2390544572113064 + }, + "test": { + "rmse": 0.25549556988031324, + "mae": 0.17997059226036072, + "r2": 0.5539265871047974, + "score": -0.25549556988031324 + } + }, + "time": "0:01:08.875317", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015188" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0.toml new file mode 100644 index 000000000..4f90bab1a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..701ca6f99 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2221833946825238, + "mae": 0.11801641434431076, + "r2": 0.6846178770065308, + "score": -0.2221833946825238 + }, + "val": { + "rmse": 0.23516710262665957, + "mae": 0.12856413424015045, + "r2": 0.6305935978889465, + "score": -0.23516710262665957 + }, + "test": { + "rmse": 0.2392893867678542, + "mae": 0.15676388144493103, + "r2": 0.6087210774421692, + "score": -0.2392893867678542 + } + }, + "time": "0:01:08.875317", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..571745d70 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22488731105453647, + "mae": 0.1187836155295372, + "r2": 0.6768949031829834, + "score": -0.22488731105453647 + }, + "val": { + "rmse": 0.23362806029940672, + "mae": 0.1260843127965927, + "r2": 0.6354129314422607, + "score": -0.23362806029940672 + }, + "test": { + "rmse": 0.23621330151542805, + "mae": 0.15148833394050598, + "r2": 0.6187162399291992, + "score": -0.23621330151542805 + } + }, + "time": "0:01:08.875317", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 8, + 22, + 7, + 29, + 25, + 21 + ], + "head_selection_time": "0:00:00.116284" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..09591f06c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23007499064462367, + "mae": 0.12374904006719589, + "r2": 0.6618162393569946, + "score": -0.23007499064462367 + }, + "val": { + "rmse": 0.2390328497980867, + "mae": 0.13070335984230042, + "r2": 0.6183489561080933, + "score": -0.2390328497980867 + }, + "test": { + "rmse": 0.24063334636332392, + "mae": 0.15138967335224152, + "r2": 0.6043134927749634, + "score": -0.24063334636332392 + } + }, + "time": "0:01:09.803835", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015082" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0.toml new file mode 100644 index 000000000..3f8f8ebf0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..6f0d01a4d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22270659663864684, + "mae": 0.11821237951517105, + "r2": 0.6831307411193848, + "score": -0.22270659663864684 + }, + "val": { + "rmse": 0.23541557264636437, + "mae": 0.1285467892885208, + "r2": 0.6298125982284546, + "score": -0.23541557264636437 + }, + "test": { + "rmse": 0.23955821427368978, + "mae": 0.15599055588245392, + "r2": 0.6078413724899292, + "score": -0.23955821427368978 + } + }, + "time": "0:01:09.803835", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4f5ce1df3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,657 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2256070659700792, + "mae": 0.11890602111816406, + "r2": 0.6748234033584595, + "score": -0.2256070659700792 + }, + "val": { + "rmse": 0.2337388066681912, + "mae": 0.12727881968021393, + "r2": 0.6350672245025635, + "score": -0.2337388066681912 + }, + "test": { + "rmse": 0.2344355028261131, + "mae": 0.14499156177043915, + "r2": 0.6244339346885681, + "score": -0.2344355028261131 + } + }, + "time": "0:01:09.803835", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 29, + 8, + 0, + 19, + 22 + ], + "head_selection_time": "0:00:00.102490" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..444867c42 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2301618504668323, + "mae": 0.12545494735240936, + "r2": 0.6615608930587769, + "score": -0.2301618504668323 + }, + "val": { + "rmse": 0.23909857764435258, + "mae": 0.13463132083415985, + "r2": 0.6181390285491943, + "score": -0.23909857764435258 + }, + "test": { + "rmse": 0.24513721764174634, + "mae": 0.16250674426555634, + "r2": 0.5893629789352417, + "score": -0.24513721764174634 + } + }, + "time": "0:01:09.043210", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015301" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0.toml new file mode 100644 index 000000000..8733113d2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..4b3d0d68d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2241623618168725, + "mae": 0.11926797032356262, + "r2": 0.6789746880531311, + "score": -0.2241623618168725 + }, + "val": { + "rmse": 0.23516078988949235, + "mae": 0.12952424585819244, + "r2": 0.6306134462356567, + "score": -0.23516078988949235 + }, + "test": { + "rmse": 0.24165682950771084, + "mae": 0.16096629202365875, + "r2": 0.600940465927124, + "score": -0.24165682950771084 + } + }, + "time": "0:01:09.043210", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8f4003f0b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,661 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22445249469109585, + "mae": 0.11813493072986603, + "r2": 0.6781431436538696, + "score": -0.22445249469109585 + }, + "val": { + "rmse": 0.23356265910507878, + "mae": 0.1266428679227829, + "r2": 0.6356170177459717, + "score": -0.23356265910507878 + }, + "test": { + "rmse": 0.23721029814115205, + "mae": 0.1523510068655014, + "r2": 0.6154908537864685, + "score": -0.23721029814115205 + } + }, + "time": "0:01:09.043210", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 7, + 4, + 22, + 19, + 8, + 18, + 29, + 7, + 25 + ], + "head_selection_time": "0:00:00.167664" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e01bf8f42 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23893091875282987, + "mae": 0.12647393345832825, + "r2": 0.6352808475494385, + "score": -0.23893091875282987 + }, + "val": { + "rmse": 0.23785345573254446, + "mae": 0.13102787733078003, + "r2": 0.6221058368682861, + "score": -0.23785345573254446 + }, + "test": { + "rmse": 0.23747813004889556, + "mae": 0.14464227855205536, + "r2": 0.6146221160888672, + "score": -0.23747813004889556 + } + }, + "time": "0:01:01.731990", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.015297" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0.toml new file mode 100644 index 000000000..d87358e44 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..ba8bcae09 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23258982929101366, + "mae": 0.1219717338681221, + "r2": 0.6543828248977661, + "score": -0.23258982929101366 + }, + "val": { + "rmse": 0.23537261362228468, + "mae": 0.12668848037719727, + "r2": 0.6299476623535156, + "score": -0.23537261362228468 + }, + "test": { + "rmse": 0.23578993148696667, + "mae": 0.14991270005702972, + "r2": 0.6200817823410034, + "score": -0.23578993148696667 + } + }, + "time": "0:01:01.731990", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3d0694006 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p0001_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,661 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2347629753255239, + "mae": 0.12291525304317474, + "r2": 0.6478942632675171, + "score": -0.2347629753255239 + }, + "val": { + "rmse": 0.23340410361335404, + "mae": 0.12670208513736725, + "r2": 0.6361116170883179, + "score": -0.23340410361335404 + }, + "test": { + "rmse": 0.2382911617531396, + "mae": 0.15507616102695465, + "r2": 0.6119787693023682, + "score": -0.2382911617531396 + } + }, + "time": "0:01:01.731990", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29, + 25, + 28, + 0, + 16, + 21, + 4, + 25, + 1, + 6 + ], + "head_selection_time": "0:00:00.165656" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..02606d789 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2282046786382406, + "mae": 0.12737023830413818, + "r2": 0.6672922372817993, + "score": -0.2282046786382406 + }, + "val": { + "rmse": 0.2390544572113064, + "mae": 0.14095881581306458, + "r2": 0.6182799935340881, + "score": -0.2390544572113064 + }, + "test": { + "rmse": 0.25549556988031324, + "mae": 0.17997059226036072, + "r2": 0.5539265871047974, + "score": -0.25549556988031324 + } + }, + "time": "0:01:06.666597", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015150" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0.toml new file mode 100644 index 000000000..97bb8b676 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..3ffd171d4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2221833946825238, + "mae": 0.11801641434431076, + "r2": 0.6846178770065308, + "score": -0.2221833946825238 + }, + "val": { + "rmse": 0.23516710262665957, + "mae": 0.12856413424015045, + "r2": 0.6305935978889465, + "score": -0.23516710262665957 + }, + "test": { + "rmse": 0.2392893867678542, + "mae": 0.15676388144493103, + "r2": 0.6087210774421692, + "score": -0.2392893867678542 + } + }, + "time": "0:01:06.666597", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d2bc4db76 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22488731105453647, + "mae": 0.1187836155295372, + "r2": 0.6768949031829834, + "score": -0.22488731105453647 + }, + "val": { + "rmse": 0.23362806029940672, + "mae": 0.1260843127965927, + "r2": 0.6354129314422607, + "score": -0.23362806029940672 + }, + "test": { + "rmse": 0.23621330151542805, + "mae": 0.15148833394050598, + "r2": 0.6187162399291992, + "score": -0.23621330151542805 + } + }, + "time": "0:01:06.666597", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 8, + 22, + 7, + 29, + 25, + 21 + ], + "head_selection_time": "0:00:00.116357" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e2a883962 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23007499064462367, + "mae": 0.12374904006719589, + "r2": 0.6618162393569946, + "score": -0.23007499064462367 + }, + "val": { + "rmse": 0.2390328497980867, + "mae": 0.13070335984230042, + "r2": 0.6183489561080933, + "score": -0.2390328497980867 + }, + "test": { + "rmse": 0.24063334636332392, + "mae": 0.15138967335224152, + "r2": 0.6043134927749634, + "score": -0.24063334636332392 + } + }, + "time": "0:01:08.046540", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015673" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0.toml new file mode 100644 index 000000000..af87226ff --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..b67ad5b90 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22270659663864684, + "mae": 0.11821237951517105, + "r2": 0.6831307411193848, + "score": -0.22270659663864684 + }, + "val": { + "rmse": 0.23541557264636437, + "mae": 0.1285467892885208, + "r2": 0.6298125982284546, + "score": -0.23541557264636437 + }, + "test": { + "rmse": 0.23955821427368978, + "mae": 0.15599055588245392, + "r2": 0.6078413724899292, + "score": -0.23955821427368978 + } + }, + "time": "0:01:08.046540", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c9e5cee60 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,657 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2256070659700792, + "mae": 0.11890602111816406, + "r2": 0.6748234033584595, + "score": -0.2256070659700792 + }, + "val": { + "rmse": 0.2337388066681912, + "mae": 0.12727881968021393, + "r2": 0.6350672245025635, + "score": -0.2337388066681912 + }, + "test": { + "rmse": 0.2344355028261131, + "mae": 0.14499156177043915, + "r2": 0.6244339346885681, + "score": -0.2344355028261131 + } + }, + "time": "0:01:08.046540", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 29, + 8, + 0, + 19, + 22 + ], + "head_selection_time": "0:00:00.104607" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..82c6a3181 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2301618504668323, + "mae": 0.12545494735240936, + "r2": 0.6615608930587769, + "score": -0.2301618504668323 + }, + "val": { + "rmse": 0.23909857764435258, + "mae": 0.13463132083415985, + "r2": 0.6181390285491943, + "score": -0.23909857764435258 + }, + "test": { + "rmse": 0.24513721764174634, + "mae": 0.16250674426555634, + "r2": 0.5893629789352417, + "score": -0.24513721764174634 + } + }, + "time": "0:01:07.636035", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015220" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0.toml new file mode 100644 index 000000000..46371ee6c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..26e59da91 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2241623618168725, + "mae": 0.11926797032356262, + "r2": 0.6789746880531311, + "score": -0.2241623618168725 + }, + "val": { + "rmse": 0.23516078988949235, + "mae": 0.12952424585819244, + "r2": 0.6306134462356567, + "score": -0.23516078988949235 + }, + "test": { + "rmse": 0.24165682950771084, + "mae": 0.16096629202365875, + "r2": 0.600940465927124, + "score": -0.24165682950771084 + } + }, + "time": "0:01:07.636035", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7140eede2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,661 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22445249469109585, + "mae": 0.11813493072986603, + "r2": 0.6781431436538696, + "score": -0.22445249469109585 + }, + "val": { + "rmse": 0.23356265910507878, + "mae": 0.1266428679227829, + "r2": 0.6356170177459717, + "score": -0.23356265910507878 + }, + "test": { + "rmse": 0.23721029814115205, + "mae": 0.1523510068655014, + "r2": 0.6154908537864685, + "score": -0.23721029814115205 + } + }, + "time": "0:01:07.636035", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 7, + 4, + 22, + 19, + 8, + 18, + 29, + 7, + 25 + ], + "head_selection_time": "0:00:00.161310" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3e9d2582e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23893091875282987, + "mae": 0.12647393345832825, + "r2": 0.6352808475494385, + "score": -0.23893091875282987 + }, + "val": { + "rmse": 0.23785345573254446, + "mae": 0.13102787733078003, + "r2": 0.6221058368682861, + "score": -0.23785345573254446 + }, + "test": { + "rmse": 0.23747813004889556, + "mae": 0.14464227855205536, + "r2": 0.6146221160888672, + "score": -0.23747813004889556 + } + }, + "time": "0:01:03.090464", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.015272" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0.toml new file mode 100644 index 000000000..0e85933ea --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..f32ddca60 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23258982929101366, + "mae": 0.1219717338681221, + "r2": 0.6543828248977661, + "score": -0.23258982929101366 + }, + "val": { + "rmse": 0.23537261362228468, + "mae": 0.12668848037719727, + "r2": 0.6299476623535156, + "score": -0.23537261362228468 + }, + "test": { + "rmse": 0.23578993148696667, + "mae": 0.14991270005702972, + "r2": 0.6200817823410034, + "score": -0.23578993148696667 + } + }, + "time": "0:01:03.090464", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3e54a594f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n0p001_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,661 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2347629753255239, + "mae": 0.12291525304317474, + "r2": 0.6478942632675171, + "score": -0.2347629753255239 + }, + "val": { + "rmse": 0.23340410361335404, + "mae": 0.12670208513736725, + "r2": 0.6361116170883179, + "score": -0.23340410361335404 + }, + "test": { + "rmse": 0.2382911617531396, + "mae": 0.15507616102695465, + "r2": 0.6119787693023682, + "score": -0.2382911617531396 + } + }, + "time": "0:01:03.090464", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29, + 25, + 28, + 0, + 16, + 21, + 4, + 25, + 1, + 6 + ], + "head_selection_time": "0:00:00.158927" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1a53b3e6a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2282046786382406, + "mae": 0.12737023830413818, + "r2": 0.6672922372817993, + "score": -0.2282046786382406 + }, + "val": { + "rmse": 0.2390544572113064, + "mae": 0.14095881581306458, + "r2": 0.6182799935340881, + "score": -0.2390544572113064 + }, + "test": { + "rmse": 0.25549556988031324, + "mae": 0.17997059226036072, + "r2": 0.5539265871047974, + "score": -0.25549556988031324 + } + }, + "time": "0:01:08.087798", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015177" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0.toml new file mode 100644 index 000000000..92f3cea1e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..e01a1f584 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2221833946825238, + "mae": 0.11801641434431076, + "r2": 0.6846178770065308, + "score": -0.2221833946825238 + }, + "val": { + "rmse": 0.23516710262665957, + "mae": 0.12856413424015045, + "r2": 0.6305935978889465, + "score": -0.23516710262665957 + }, + "test": { + "rmse": 0.2392893867678542, + "mae": 0.15676388144493103, + "r2": 0.6087210774421692, + "score": -0.2392893867678542 + } + }, + "time": "0:01:08.087798", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..485b0677c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22488731105453647, + "mae": 0.1187836155295372, + "r2": 0.6768949031829834, + "score": -0.22488731105453647 + }, + "val": { + "rmse": 0.23362806029940672, + "mae": 0.1260843127965927, + "r2": 0.6354129314422607, + "score": -0.23362806029940672 + }, + "test": { + "rmse": 0.23621330151542805, + "mae": 0.15148833394050598, + "r2": 0.6187162399291992, + "score": -0.23621330151542805 + } + }, + "time": "0:01:08.087798", + "diversity": { + "train": { + "mean_centered_corr": -0.019289543852210045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04795046150684357 + }, + "val": { + "mean_centered_corr": -0.019224567338824272, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06529936194419861 + }, + "test": { + "mean_centered_corr": -0.01879088766872883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741392940282822 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 8, + 22, + 7, + 29, + 25, + 21 + ], + "head_selection_time": "0:00:00.115387" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e6a148498 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23007499064462367, + "mae": 0.12374904006719589, + "r2": 0.6618162393569946, + "score": -0.23007499064462367 + }, + "val": { + "rmse": 0.2390328497980867, + "mae": 0.13070335984230042, + "r2": 0.6183489561080933, + "score": -0.2390328497980867 + }, + "test": { + "rmse": 0.24063334636332392, + "mae": 0.15138967335224152, + "r2": 0.6043134927749634, + "score": -0.24063334636332392 + } + }, + "time": "0:01:06.267932", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015159" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0.toml new file mode 100644 index 000000000..82e1f7182 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..b8a5dad5e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22270659663864684, + "mae": 0.11821237951517105, + "r2": 0.6831307411193848, + "score": -0.22270659663864684 + }, + "val": { + "rmse": 0.23541557264636437, + "mae": 0.1285467892885208, + "r2": 0.6298125982284546, + "score": -0.23541557264636437 + }, + "test": { + "rmse": 0.23955821427368978, + "mae": 0.15599055588245392, + "r2": 0.6078413724899292, + "score": -0.23955821427368978 + } + }, + "time": "0:01:06.267932", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0ed641c86 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,657 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2256070659700792, + "mae": 0.11890602111816406, + "r2": 0.6748234033584595, + "score": -0.2256070659700792 + }, + "val": { + "rmse": 0.2337388066681912, + "mae": 0.12727881968021393, + "r2": 0.6350672245025635, + "score": -0.2337388066681912 + }, + "test": { + "rmse": 0.2344355028261131, + "mae": 0.14499156177043915, + "r2": 0.6244339346885681, + "score": -0.2344355028261131 + } + }, + "time": "0:01:06.267932", + "diversity": { + "train": { + "mean_centered_corr": -0.018535790964961052, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048687759786844254 + }, + "val": { + "mean_centered_corr": -0.019753795117139816, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561429798603058 + }, + "test": { + "mean_centered_corr": -0.01759418286383152, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573547542095184 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 29, + 8, + 0, + 19, + 22 + ], + "head_selection_time": "0:00:00.100716" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a84a97944 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2301618504668323, + "mae": 0.12545494735240936, + "r2": 0.6615608930587769, + "score": -0.2301618504668323 + }, + "val": { + "rmse": 0.23909857764435258, + "mae": 0.13463132083415985, + "r2": 0.6181390285491943, + "score": -0.23909857764435258 + }, + "test": { + "rmse": 0.24513721764174634, + "mae": 0.16250674426555634, + "r2": 0.5893629789352417, + "score": -0.24513721764174634 + } + }, + "time": "0:01:06.451437", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.014906" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0.toml new file mode 100644 index 000000000..11cc63c07 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..4d2447da1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2241623618168725, + "mae": 0.11926797032356262, + "r2": 0.6789746880531311, + "score": -0.2241623618168725 + }, + "val": { + "rmse": 0.23516078988949235, + "mae": 0.12952424585819244, + "r2": 0.6306134462356567, + "score": -0.23516078988949235 + }, + "test": { + "rmse": 0.24165682950771084, + "mae": 0.16096629202365875, + "r2": 0.600940465927124, + "score": -0.24165682950771084 + } + }, + "time": "0:01:06.451437", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..71f1b5335 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,661 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22445249469109585, + "mae": 0.11813493072986603, + "r2": 0.6781431436538696, + "score": -0.22445249469109585 + }, + "val": { + "rmse": 0.23356265910507878, + "mae": 0.1266428679227829, + "r2": 0.6356170177459717, + "score": -0.23356265910507878 + }, + "test": { + "rmse": 0.23721029814115205, + "mae": 0.1523510068655014, + "r2": 0.6154908537864685, + "score": -0.23721029814115205 + } + }, + "time": "0:01:06.451437", + "diversity": { + "train": { + "mean_centered_corr": -0.01836317405104637, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04760168120265007 + }, + "val": { + "mean_centered_corr": -0.01840268261730671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696823239326477 + }, + "test": { + "mean_centered_corr": -0.018157226964831352, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09846261888742447 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 7, + 4, + 22, + 19, + 8, + 18, + 29, + 7, + 25 + ], + "head_selection_time": "0:00:00.157239" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..d60489982 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,652 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23893091875282987, + "mae": 0.12647393345832825, + "r2": 0.6352808475494385, + "score": -0.23893091875282987 + }, + "val": { + "rmse": 0.23785345573254446, + "mae": 0.13102787733078003, + "r2": 0.6221058368682861, + "score": -0.23785345573254446 + }, + "test": { + "rmse": 0.23747813004889556, + "mae": 0.14464227855205536, + "r2": 0.6146221160888672, + "score": -0.23747813004889556 + } + }, + "time": "0:01:03.527749", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.015077" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0.toml new file mode 100644 index 000000000..cb443d190 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..a1773ffe5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-evaluation/0/report.json @@ -0,0 +1,648 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23258982929101366, + "mae": 0.1219717338681221, + "r2": 0.6543828248977661, + "score": -0.23258982929101366 + }, + "val": { + "rmse": 0.23537261362228468, + "mae": 0.12668848037719727, + "r2": 0.6299476623535156, + "score": -0.23537261362228468 + }, + "test": { + "rmse": 0.23578993148696667, + "mae": 0.14991270005702972, + "r2": 0.6200817823410034, + "score": -0.23578993148696667 + } + }, + "time": "0:01:03.527749", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1ff6458ff --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r1_n1em05_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,661 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2347629753255239, + "mae": 0.12291525304317474, + "r2": 0.6478942632675171, + "score": -0.2347629753255239 + }, + "val": { + "rmse": 0.23340410361335404, + "mae": 0.12670208513736725, + "r2": 0.6361116170883179, + "score": -0.23340410361335404 + }, + "test": { + "rmse": 0.2382911617531396, + "mae": 0.15507616102695465, + "r2": 0.6119787693023682, + "score": -0.2382911617531396 + } + }, + "time": "0:01:03.527749", + "diversity": { + "train": { + "mean_centered_corr": -0.019388988614082336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785213619470596 + }, + "val": { + "mean_centered_corr": -0.01973061077296734, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109660863876343 + }, + "test": { + "mean_centered_corr": -0.019816331565380096, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08115967363119125 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29, + 25, + 28, + 0, + 16, + 21, + 4, + 25, + 1, + 6 + ], + "head_selection_time": "0:00:00.158406" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..82a0d9f14 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22820973096491176, + "mae": 0.12737685441970825, + "r2": 0.667277455329895, + "score": -0.22820973096491176 + }, + "val": { + "rmse": 0.23905700508983188, + "mae": 0.14095449447631836, + "r2": 0.6182718276977539, + "score": -0.23905700508983188 + }, + "test": { + "rmse": 0.2554892709638668, + "mae": 0.17996472120285034, + "r2": 0.5539485812187195, + "score": -0.2554892709638668 + } + }, + "time": "0:01:27.923469", + "diversity": { + "train": { + "mean_centered_corr": -0.019317513331770897, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04794894531369209 + }, + "val": { + "mean_centered_corr": -0.019240468740463257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.065296970307827 + }, + "test": { + "mean_centered_corr": -0.018747208639979362, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741389214992523 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.016127" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0.toml new file mode 100644 index 000000000..12f1a9ea9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..650fcbc1c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22218533123207995, + "mae": 0.11801747977733612, + "r2": 0.6846123933792114, + "score": -0.22218533123207995 + }, + "val": { + "rmse": 0.23516964509902982, + "mae": 0.1285666972398758, + "r2": 0.6305856704711914, + "score": -0.23516964509902982 + }, + "test": { + "rmse": 0.23928895864366434, + "mae": 0.15676262974739075, + "r2": 0.608722448348999, + "score": -0.23928895864366434 + } + }, + "time": "0:01:27.923469", + "diversity": { + "train": { + "mean_centered_corr": -0.019317513331770897, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04794894531369209 + }, + "val": { + "mean_centered_corr": -0.019240468740463257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.065296970307827 + }, + "test": { + "mean_centered_corr": -0.018747208639979362, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741389214992523 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3f00e7ab6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,664 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22489126180626096, + "mae": 0.11878597736358643, + "r2": 0.6768835783004761, + "score": -0.22489126180626096 + }, + "val": { + "rmse": 0.23363252496566134, + "mae": 0.12608498334884644, + "r2": 0.6353989839553833, + "score": -0.23363252496566134 + }, + "test": { + "rmse": 0.2362188764537236, + "mae": 0.15149326622486115, + "r2": 0.618698239326477, + "score": -0.2362188764537236 + } + }, + "time": "0:01:27.923469", + "diversity": { + "train": { + "mean_centered_corr": -0.019317513331770897, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04794894531369209 + }, + "val": { + "mean_centered_corr": -0.019240468740463257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.065296970307827 + }, + "test": { + "mean_centered_corr": -0.018747208639979362, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741389214992523 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 8, + 22, + 7, + 29, + 25, + 21 + ], + "head_selection_time": "0:00:00.117264" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..860fa6472 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23007882803014523, + "mae": 0.12374971061944962, + "r2": 0.6618049740791321, + "score": -0.23007882803014523 + }, + "val": { + "rmse": 0.23901989843968774, + "mae": 0.13069558143615723, + "r2": 0.6183903217315674, + "score": -0.23901989843968774 + }, + "test": { + "rmse": 0.24063495640158664, + "mae": 0.1513846516609192, + "r2": 0.6043082475662231, + "score": -0.24063495640158664 + } + }, + "time": "0:01:28.376248", + "diversity": { + "train": { + "mean_centered_corr": -0.018541699275374413, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04868823289871216 + }, + "val": { + "mean_centered_corr": -0.019729774445295334, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561318784952164 + }, + "test": { + "mean_centered_corr": -0.01752825267612934, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573257714509964 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015332" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0.toml new file mode 100644 index 000000000..ae4744a5f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..0b121938f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22270596936228843, + "mae": 0.11821265518665314, + "r2": 0.6831325888633728, + "score": -0.22270596936228843 + }, + "val": { + "rmse": 0.23541076200529618, + "mae": 0.1285439282655716, + "r2": 0.6298277378082275, + "score": -0.23541076200529618 + }, + "test": { + "rmse": 0.2395584319829453, + "mae": 0.15599201619625092, + "r2": 0.6078406572341919, + "score": -0.2395584319829453 + } + }, + "time": "0:01:28.376248", + "diversity": { + "train": { + "mean_centered_corr": -0.018541699275374413, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04868823289871216 + }, + "val": { + "mean_centered_corr": -0.019729774445295334, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561318784952164 + }, + "test": { + "mean_centered_corr": -0.01752825267612934, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573257714509964 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5936c02a0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,663 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22560705771393183, + "mae": 0.11890538036823273, + "r2": 0.6748234033584595, + "score": -0.22560705771393183 + }, + "val": { + "rmse": 0.23373660723675202, + "mae": 0.1272813230752945, + "r2": 0.6350740790367126, + "score": -0.23373660723675202 + }, + "test": { + "rmse": 0.23443243594536417, + "mae": 0.14499256014823914, + "r2": 0.624443769454956, + "score": -0.23443243594536417 + } + }, + "time": "0:01:28.376248", + "diversity": { + "train": { + "mean_centered_corr": -0.018541699275374413, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04868823289871216 + }, + "val": { + "mean_centered_corr": -0.019729774445295334, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561318784952164 + }, + "test": { + "mean_centered_corr": -0.01752825267612934, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573257714509964 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 29, + 8, + 0, + 19, + 22 + ], + "head_selection_time": "0:00:00.101988" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3d02dfe64 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23015652536762907, + "mae": 0.12545116245746613, + "r2": 0.661576509475708, + "score": -0.23015652536762907 + }, + "val": { + "rmse": 0.239085684381371, + "mae": 0.13462616503238678, + "r2": 0.6181802749633789, + "score": -0.239085684381371 + }, + "test": { + "rmse": 0.24513387433279307, + "mae": 0.16250139474868774, + "r2": 0.5893741846084595, + "score": -0.24513387433279307 + } + }, + "time": "0:01:28.822139", + "diversity": { + "train": { + "mean_centered_corr": -0.01839604787528515, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0476016141474247 + }, + "val": { + "mean_centered_corr": -0.0184602253139019, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696488708257675 + }, + "test": { + "mean_centered_corr": -0.018153522163629532, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09845950454473495 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015540" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0.toml new file mode 100644 index 000000000..2bf833500 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..f30d39703 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22416379102176556, + "mae": 0.11926887929439545, + "r2": 0.6789705753326416, + "score": -0.22416379102176556 + }, + "val": { + "rmse": 0.23516104335271928, + "mae": 0.12952451407909393, + "r2": 0.6306126713752747, + "score": -0.23516104335271928 + }, + "test": { + "rmse": 0.24165658285763553, + "mae": 0.1609659045934677, + "r2": 0.6009412407875061, + "score": -0.24165658285763553 + } + }, + "time": "0:01:28.822139", + "diversity": { + "train": { + "mean_centered_corr": -0.01839604787528515, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0476016141474247 + }, + "val": { + "mean_centered_corr": -0.0184602253139019, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696488708257675 + }, + "test": { + "mean_centered_corr": -0.018153522163629532, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09845950454473495 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b2716d4c8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,667 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22445475190340167, + "mae": 0.11813689768314362, + "r2": 0.6781366467475891, + "score": -0.22445475190340167 + }, + "val": { + "rmse": 0.23356035433981295, + "mae": 0.1266433298587799, + "r2": 0.6356241703033447, + "score": -0.23356035433981295 + }, + "test": { + "rmse": 0.2372102431750833, + "mae": 0.15235204994678497, + "r2": 0.6154910326004028, + "score": -0.2372102431750833 + } + }, + "time": "0:01:28.822139", + "diversity": { + "train": { + "mean_centered_corr": -0.01839604787528515, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0476016141474247 + }, + "val": { + "mean_centered_corr": -0.0184602253139019, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696488708257675 + }, + "test": { + "mean_centered_corr": -0.018153522163629532, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09845950454473495 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 7, + 4, + 22, + 19, + 8, + 18, + 7, + 29, + 25 + ], + "head_selection_time": "0:00:00.163829" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..17b27f8c1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23893329644403358, + "mae": 0.12647750973701477, + "r2": 0.6352735757827759, + "score": -0.23893329644403358 + }, + "val": { + "rmse": 0.23785555444789067, + "mae": 0.1310475766658783, + "r2": 0.6220991611480713, + "score": -0.23785555444789067 + }, + "test": { + "rmse": 0.23748897727684737, + "mae": 0.1446538269519806, + "r2": 0.6145868301391602, + "score": -0.23748897727684737 + } + }, + "time": "0:01:22.606690", + "diversity": { + "train": { + "mean_centered_corr": -0.019367322325706482, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785257205367088 + }, + "val": { + "mean_centered_corr": -0.0196691807359457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109632924199104 + }, + "test": { + "mean_centered_corr": -0.01985469087958336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08116098493337631 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.015212" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0.toml new file mode 100644 index 000000000..93fff1a04 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..61f313a8f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2325909984973762, + "mae": 0.12197329849004745, + "r2": 0.6543793678283691, + "score": -0.2325909984973762 + }, + "val": { + "rmse": 0.23537506682607195, + "mae": 0.12669186294078827, + "r2": 0.6299399733543396, + "score": -0.23537506682607195 + }, + "test": { + "rmse": 0.23579029486809283, + "mae": 0.14991408586502075, + "r2": 0.6200805902481079, + "score": -0.23579029486809283 + } + }, + "time": "0:01:22.606690", + "diversity": { + "train": { + "mean_centered_corr": -0.019367322325706482, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785257205367088 + }, + "val": { + "mean_centered_corr": -0.0196691807359457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109632924199104 + }, + "test": { + "mean_centered_corr": -0.01985469087958336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08116098493337631 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..eaf94028d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,667 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23476379254176588, + "mae": 0.12291711568832397, + "r2": 0.6478918194770813, + "score": -0.23476379254176588 + }, + "val": { + "rmse": 0.23340542834677785, + "mae": 0.12670302391052246, + "r2": 0.6361074447631836, + "score": -0.23340542834677785 + }, + "test": { + "rmse": 0.23829137280333715, + "mae": 0.15507613122463226, + "r2": 0.6119781136512756, + "score": -0.23829137280333715 + } + }, + "time": "0:01:22.606690", + "diversity": { + "train": { + "mean_centered_corr": -0.019367322325706482, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785257205367088 + }, + "val": { + "mean_centered_corr": -0.0196691807359457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109632924199104 + }, + "test": { + "mean_centered_corr": -0.01985469087958336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08116098493337631 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29, + 25, + 28, + 0, + 16, + 21, + 4, + 25, + 1, + 6 + ], + "head_selection_time": "0:00:00.171698" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ee0b0e618 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22949952458766654, + "mae": 0.1217280700802803, + "r2": 0.6635059118270874, + "score": -0.22949952458766654 + }, + "val": { + "rmse": 0.23798803361544055, + "mae": 0.1329043060541153, + "r2": 0.6216781139373779, + "score": -0.23798803361544055 + }, + "test": { + "rmse": 0.23971064008247925, + "mae": 0.14922766387462616, + "r2": 0.6073422431945801, + "score": -0.23971064008247925 + } + }, + "time": "0:01:27.946940", + "diversity": { + "train": { + "mean_centered_corr": -0.018655812367796898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04690698906779289 + }, + "val": { + "mean_centered_corr": -0.019464505836367607, + "mean_pairwise_disagreement": NaN, + "member_std": 0.062106743454933167 + }, + "test": { + "mean_centered_corr": -0.01887187734246254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08776523917913437 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015638" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0.toml new file mode 100644 index 000000000..fb28a1b73 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..643ffc8ef --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22223358862007636, + "mae": 0.11760532855987549, + "r2": 0.6844753623008728, + "score": -0.22223358862007636 + }, + "val": { + "rmse": 0.23492518063329523, + "mae": 0.12833383679389954, + "r2": 0.6313532590866089, + "score": -0.23492518063329523 + }, + "test": { + "rmse": 0.23863885185928208, + "mae": 0.15527929365634918, + "r2": 0.610845685005188, + "score": -0.23863885185928208 + } + }, + "time": "0:01:27.946940", + "diversity": { + "train": { + "mean_centered_corr": -0.018655812367796898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04690698906779289 + }, + "val": { + "mean_centered_corr": -0.019464505836367607, + "mean_pairwise_disagreement": NaN, + "member_std": 0.062106743454933167 + }, + "test": { + "mean_centered_corr": -0.01887187734246254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08776523917913437 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..98afe708a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,663 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2249769437420875, + "mae": 0.11708378046751022, + "r2": 0.6766372919082642, + "score": -0.2249769437420875 + }, + "val": { + "rmse": 0.23342205869781213, + "mae": 0.12662439048290253, + "r2": 0.636055588722229, + "score": -0.23342205869781213 + }, + "test": { + "rmse": 0.23945135735112927, + "mae": 0.1564028114080429, + "r2": 0.608191192150116, + "score": -0.23945135735112927 + } + }, + "time": "0:01:27.946940", + "diversity": { + "train": { + "mean_centered_corr": -0.018655812367796898, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04690698906779289 + }, + "val": { + "mean_centered_corr": -0.019464505836367607, + "mean_pairwise_disagreement": NaN, + "member_std": 0.062106743454933167 + }, + "test": { + "mean_centered_corr": -0.01887187734246254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08776523917913437 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 4, + 8, + 25, + 22, + 7 + ], + "head_selection_time": "0:00:00.101885" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..94b39e407 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23627931705607993, + "mae": 0.12559163570404053, + "r2": 0.6433310508728027, + "score": -0.23627931705607993 + }, + "val": { + "rmse": 0.23811064537966578, + "mae": 0.13582253456115723, + "r2": 0.6212881803512573, + "score": -0.23811064537966578 + }, + "test": { + "rmse": 0.24336730552230365, + "mae": 0.16211101412773132, + "r2": 0.5952712297439575, + "score": -0.24336730552230365 + } + }, + "time": "0:01:24.069070", + "diversity": { + "train": { + "mean_centered_corr": -0.018960177898406982, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04602884501218796 + }, + "val": { + "mean_centered_corr": -0.01958228461444378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06140345335006714 + }, + "test": { + "mean_centered_corr": -0.017305321991443634, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08274664729833603 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.015270" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0.toml new file mode 100644 index 000000000..ba5d2c125 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..ee3e54b8c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23267118722425764, + "mae": 0.12135601043701172, + "r2": 0.6541409492492676, + "score": -0.23267118722425764 + }, + "val": { + "rmse": 0.23501775344788706, + "mae": 0.12565036118030548, + "r2": 0.6310626268386841, + "score": -0.23501775344788706 + }, + "test": { + "rmse": 0.23276117546354494, + "mae": 0.1444091498851776, + "r2": 0.6297793388366699, + "score": -0.23276117546354494 + } + }, + "time": "0:01:24.069070", + "diversity": { + "train": { + "mean_centered_corr": -0.018960177898406982, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04602884501218796 + }, + "val": { + "mean_centered_corr": -0.01958228461444378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06140345335006714 + }, + "test": { + "mean_centered_corr": -0.017305321991443634, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08274664729833603 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d116f6601 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,668 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23364578292705543, + "mae": 0.12276748567819595, + "r2": 0.6512374877929688, + "score": -0.23364578292705543 + }, + "val": { + "rmse": 0.23317303008072252, + "mae": 0.12586387991905212, + "r2": 0.6368317604064941, + "score": -0.23317303008072252 + }, + "test": { + "rmse": 0.23484075680143124, + "mae": 0.14914271235466003, + "r2": 0.6231343746185303, + "score": -0.23484075680143124 + } + }, + "time": "0:01:24.069070", + "diversity": { + "train": { + "mean_centered_corr": -0.018960177898406982, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04602884501218796 + }, + "val": { + "mean_centered_corr": -0.01958228461444378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06140345335006714 + }, + "test": { + "mean_centered_corr": -0.017305321991443634, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08274664729833603 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 21, + 25, + 4, + 29, + 0, + 27, + 6, + 25, + 21, + 19 + ], + "head_selection_time": "0:00:00.174870" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..33dd4e5f3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23695884432306313, + "mae": 0.12806488573551178, + "r2": 0.6412765979766846, + "score": -0.23695884432306313 + }, + "val": { + "rmse": 0.23912916812165463, + "mae": 0.1332480013370514, + "r2": 0.6180413365364075, + "score": -0.23912916812165463 + }, + "test": { + "rmse": 0.23873190398066874, + "mae": 0.14833378791809082, + "r2": 0.6105421185493469, + "score": -0.23873190398066874 + } + }, + "time": "0:01:23.755099", + "diversity": { + "train": { + "mean_centered_corr": -0.018932092934846878, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046318743377923965 + }, + "val": { + "mean_centered_corr": -0.020040452480316162, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06007201597094536 + }, + "test": { + "mean_centered_corr": -0.017600608989596367, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0855417475104332 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.015574" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0.toml new file mode 100644 index 000000000..05d2b7359 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..2015657ce --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23164681624168468, + "mae": 0.12229707092046738, + "r2": 0.6571797132492065, + "score": -0.23164681624168468 + }, + "val": { + "rmse": 0.23514059116024372, + "mae": 0.12843270599842072, + "r2": 0.6306768655776978, + "score": -0.23514059116024372 + }, + "test": { + "rmse": 0.23749547910151964, + "mae": 0.15404725074768066, + "r2": 0.6145657300949097, + "score": -0.23749547910151964 + } + }, + "time": "0:01:23.755099", + "diversity": { + "train": { + "mean_centered_corr": -0.018932092934846878, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046318743377923965 + }, + "val": { + "mean_centered_corr": -0.020040452480316162, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06007201597094536 + }, + "test": { + "mean_centered_corr": -0.017600608989596367, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0855417475104332 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..eb460972f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,667 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.232880388512877, + "mae": 0.12257546931505203, + "r2": 0.653518795967102, + "score": -0.232880388512877 + }, + "val": { + "rmse": 0.2333667286677465, + "mae": 0.1266324818134308, + "r2": 0.636228084564209, + "score": -0.2333667286677465 + }, + "test": { + "rmse": 0.2358601642328826, + "mae": 0.15086041390895844, + "r2": 0.6198554039001465, + "score": -0.2358601642328826 + } + }, + "time": "0:01:23.755099", + "diversity": { + "train": { + "mean_centered_corr": -0.018932092934846878, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046318743377923965 + }, + "val": { + "mean_centered_corr": -0.020040452480316162, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06007201597094536 + }, + "test": { + "mean_centered_corr": -0.017600608989596367, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0855417475104332 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29, + 28, + 25, + 10, + 19, + 0, + 21, + 6, + 25, + 28 + ], + "head_selection_time": "0:00:00.163588" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..be1eea845 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23145121152418155, + "mae": 0.1271854192018509, + "r2": 0.657758355140686, + "score": -0.23145121152418155 + }, + "val": { + "rmse": 0.23792787050471845, + "mae": 0.13454143702983856, + "r2": 0.6218693256378174, + "score": -0.23792787050471845 + }, + "test": { + "rmse": 0.24466013756612576, + "mae": 0.16113941371440887, + "r2": 0.5909597873687744, + "score": -0.24466013756612576 + } + }, + "time": "0:01:29.402227", + "diversity": { + "train": { + "mean_centered_corr": -0.018917854875326157, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04637383669614792 + }, + "val": { + "mean_centered_corr": -0.019510701298713684, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06465298682451248 + }, + "test": { + "mean_centered_corr": -0.01702488213777542, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09602542966604233 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015386" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0.toml new file mode 100644 index 000000000..f13bcf99c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..59379db23 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2243097379869937, + "mae": 0.12035604566335678, + "r2": 0.6785524487495422, + "score": -0.2243097379869937 + }, + "val": { + "rmse": 0.23506069808417399, + "mae": 0.12883473932743073, + "r2": 0.6309278011322021, + "score": -0.23506069808417399 + }, + "test": { + "rmse": 0.24070191052163564, + "mae": 0.15875394642353058, + "r2": 0.6040880084037781, + "score": -0.24070191052163564 + } + }, + "time": "0:01:29.402227", + "diversity": { + "train": { + "mean_centered_corr": -0.018917854875326157, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04637383669614792 + }, + "val": { + "mean_centered_corr": -0.019510701298713684, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06465298682451248 + }, + "test": { + "mean_centered_corr": -0.01702488213777542, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09602542966604233 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7ef3e192f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p0001_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,663 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22678944303849974, + "mae": 0.12170526385307312, + "r2": 0.6714060306549072, + "score": -0.22678944303849974 + }, + "val": { + "rmse": 0.23322026782389782, + "mae": 0.12833212316036224, + "r2": 0.6366845369338989, + "score": -0.23322026782389782 + }, + "test": { + "rmse": 0.23659461735657236, + "mae": 0.1518249213695526, + "r2": 0.6174842119216919, + "score": -0.23659461735657236 + } + }, + "time": "0:01:29.402227", + "diversity": { + "train": { + "mean_centered_corr": -0.018917854875326157, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04637383669614792 + }, + "val": { + "mean_centered_corr": -0.019510701298713684, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06465298682451248 + }, + "test": { + "mean_centered_corr": -0.01702488213777542, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09602542966604233 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 29, + 8, + 25, + 19, + 1 + ], + "head_selection_time": "0:00:00.103468" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..41a098b0d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2282022544611775, + "mae": 0.12789320945739746, + "r2": 0.6672992706298828, + "score": -0.2282022544611775 + }, + "val": { + "rmse": 0.24061645579340615, + "mae": 0.14481423795223236, + "r2": 0.6132752895355225, + "score": -0.24061645579340615 + }, + "test": { + "rmse": 0.2630298852149786, + "mae": 0.1882508099079132, + "r2": 0.5272301435470581, + "score": -0.2630298852149786 + } + }, + "time": "0:01:29.554714", + "diversity": { + "train": { + "mean_centered_corr": -0.019097518175840378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047171760350465775 + }, + "val": { + "mean_centered_corr": -0.01874253898859024, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0679805725812912 + }, + "test": { + "mean_centered_corr": -0.016998732462525368, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08968250453472137 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015426" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0.toml new file mode 100644 index 000000000..afea6e51a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..09db8da3b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2217449394684032, + "mae": 0.11953334510326385, + "r2": 0.685861349105835, + "score": -0.2217449394684032 + }, + "val": { + "rmse": 0.23558290278789318, + "mae": 0.12969213724136353, + "r2": 0.6292861700057983, + "score": -0.23558290278789318 + }, + "test": { + "rmse": 0.24195079618816864, + "mae": 0.16031414270401, + "r2": 0.5999689102172852, + "score": -0.24195079618816864 + } + }, + "time": "0:01:29.554714", + "diversity": { + "train": { + "mean_centered_corr": -0.019097518175840378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047171760350465775 + }, + "val": { + "mean_centered_corr": -0.01874253898859024, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0679805725812912 + }, + "test": { + "mean_centered_corr": -0.016998732462525368, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08968250453472137 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5a30a8506 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,661 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22122007777848465, + "mae": 0.11975695192813873, + "r2": 0.6873466968536377, + "score": -0.22122007777848465 + }, + "val": { + "rmse": 0.23376432971584016, + "mae": 0.12689536809921265, + "r2": 0.634987473487854, + "score": -0.23376432971584016 + }, + "test": { + "rmse": 0.24301364183447563, + "mae": 0.1616802215576172, + "r2": 0.596446692943573, + "score": -0.24301364183447563 + } + }, + "time": "0:01:29.554714", + "diversity": { + "train": { + "mean_centered_corr": -0.019097518175840378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047171760350465775 + }, + "val": { + "mean_centered_corr": -0.01874253898859024, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0679805725812912 + }, + "test": { + "mean_centered_corr": -0.016998732462525368, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08968250453472137 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 21, + 25, + 19 + ], + "head_selection_time": "0:00:00.074750" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1dd4c094f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23952885281283356, + "mae": 0.12619903683662415, + "r2": 0.6334531307220459, + "score": -0.23952885281283356 + }, + "val": { + "rmse": 0.23885129531058336, + "mae": 0.13529641926288605, + "r2": 0.6189285516738892, + "score": -0.23885129531058336 + }, + "test": { + "rmse": 0.24654667666950852, + "mae": 0.16894596815109253, + "r2": 0.5846273899078369, + "score": -0.24654667666950852 + } + }, + "time": "0:01:20.972441", + "diversity": { + "train": { + "mean_centered_corr": -0.019369952380657196, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046703703701496124 + }, + "val": { + "mean_centered_corr": -0.018561262637376785, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06341557204723358 + }, + "test": { + "mean_centered_corr": -0.01718106120824814, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08355174213647842 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015400" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0.toml new file mode 100644 index 000000000..e423f0302 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..e341e65b1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23157858733487574, + "mae": 0.1211637556552887, + "r2": 0.6573815941810608, + "score": -0.23157858733487574 + }, + "val": { + "rmse": 0.23531810633502007, + "mae": 0.125682532787323, + "r2": 0.6301190853118896, + "score": -0.23531810633502007 + }, + "test": { + "rmse": 0.2327283473545216, + "mae": 0.1438872069120407, + "r2": 0.6298837661743164, + "score": -0.2327283473545216 + } + }, + "time": "0:01:20.972441", + "diversity": { + "train": { + "mean_centered_corr": -0.019369952380657196, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046703703701496124 + }, + "val": { + "mean_centered_corr": -0.018561262637376785, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06341557204723358 + }, + "test": { + "mean_centered_corr": -0.01718106120824814, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08355174213647842 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3853fa9e7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,665 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2322140894845545, + "mae": 0.12190577387809753, + "r2": 0.6554985642433167, + "score": -0.2322140894845545 + }, + "val": { + "rmse": 0.23374365968826152, + "mae": 0.12585988640785217, + "r2": 0.6350520253181458, + "score": -0.23374365968826152 + }, + "test": { + "rmse": 0.23258765102786882, + "mae": 0.14328286051750183, + "r2": 0.6303311586380005, + "score": -0.23258765102786882 + } + }, + "time": "0:01:20.972441", + "diversity": { + "train": { + "mean_centered_corr": -0.019369952380657196, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046703703701496124 + }, + "val": { + "mean_centered_corr": -0.018561262637376785, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06341557204723358 + }, + "test": { + "mean_centered_corr": -0.01718106120824814, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08355174213647842 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 21, + 0, + 25, + 29, + 6, + 19, + 28 + ], + "head_selection_time": "0:00:00.130802" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..588e93cc8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23598866476350114, + "mae": 0.13086389005184174, + "r2": 0.6442080140113831, + "score": -0.23598866476350114 + }, + "val": { + "rmse": 0.23798534123823967, + "mae": 0.13590504229068756, + "r2": 0.6216866970062256, + "score": -0.23798534123823967 + }, + "test": { + "rmse": 0.2395424764805503, + "mae": 0.1539018154144287, + "r2": 0.6078929305076599, + "score": -0.2395424764805503 + } + }, + "time": "0:01:18.608008", + "diversity": { + "train": { + "mean_centered_corr": -0.01912665367126465, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045948561280965805 + }, + "val": { + "mean_centered_corr": -0.01955518126487732, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05995342880487442 + }, + "test": { + "mean_centered_corr": -0.015723563730716705, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08670108765363693 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:00.015223" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0.toml new file mode 100644 index 000000000..4c97f04e8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..060cded4a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23216121546628654, + "mae": 0.12173006683588028, + "r2": 0.6556554436683655, + "score": -0.23216121546628654 + }, + "val": { + "rmse": 0.23480153213840635, + "mae": 0.12647123634815216, + "r2": 0.6317412257194519, + "score": -0.23480153213840635 + }, + "test": { + "rmse": 0.23483096909658374, + "mae": 0.14862899482250214, + "r2": 0.6231657266616821, + "score": -0.23483096909658374 + } + }, + "time": "0:01:18.608008", + "diversity": { + "train": { + "mean_centered_corr": -0.01912665367126465, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045948561280965805 + }, + "val": { + "mean_centered_corr": -0.01955518126487732, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05995342880487442 + }, + "test": { + "mean_centered_corr": -0.015723563730716705, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08670108765363693 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..71e10c5c5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,666 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23448659306502342, + "mae": 0.12265997380018234, + "r2": 0.6487228274345398, + "score": -0.23448659306502342 + }, + "val": { + "rmse": 0.23320259273091312, + "mae": 0.12646806240081787, + "r2": 0.6367396116256714, + "score": -0.23320259273091312 + }, + "test": { + "rmse": 0.23641549221562166, + "mae": 0.15262219309806824, + "r2": 0.61806321144104, + "score": -0.23641549221562166 + } + }, + "time": "0:01:18.608008", + "diversity": { + "train": { + "mean_centered_corr": -0.01912665367126465, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045948561280965805 + }, + "val": { + "mean_centered_corr": -0.01955518126487732, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05995342880487442 + }, + "test": { + "mean_centered_corr": -0.015723563730716705, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08670108765363693 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 1, + 28, + 0, + 29, + 21, + 25, + 1, + 4 + ], + "head_selection_time": "0:00:00.145758" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..be1e3d331 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23020776422064904, + "mae": 0.1262739598751068, + "r2": 0.6614258289337158, + "score": -0.23020776422064904 + }, + "val": { + "rmse": 0.23771204736707321, + "mae": 0.13801762461662292, + "r2": 0.6225550174713135, + "score": -0.23771204736707321 + }, + "test": { + "rmse": 0.24308994802985293, + "mae": 0.16311529278755188, + "r2": 0.5961931943893433, + "score": -0.24308994802985293 + } + }, + "time": "0:01:27.894088", + "diversity": { + "train": { + "mean_centered_corr": -0.018592098727822304, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045505620539188385 + }, + "val": { + "mean_centered_corr": -0.018926210701465607, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06432383507490158 + }, + "test": { + "mean_centered_corr": -0.012847395613789558, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09277871996164322 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.015561" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0.toml new file mode 100644 index 000000000..ff31b4334 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..fa6fecdb7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22398877861023744, + "mae": 0.1194881871342659, + "r2": 0.6794716715812683, + "score": -0.22398877861023744 + }, + "val": { + "rmse": 0.23510019641837185, + "mae": 0.12923409044742584, + "r2": 0.6308038234710693, + "score": -0.23510019641837185 + }, + "test": { + "rmse": 0.23935828921252758, + "mae": 0.15663591027259827, + "r2": 0.6084957122802734, + "score": -0.23935828921252758 + } + }, + "time": "0:01:27.894088", + "diversity": { + "train": { + "mean_centered_corr": -0.018592098727822304, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045505620539188385 + }, + "val": { + "mean_centered_corr": -0.018926210701465607, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06432383507490158 + }, + "test": { + "mean_centered_corr": -0.012847395613789558, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09277871996164322 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..47d385ff1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n0p001_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,663 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22544924405132602, + "mae": 0.12180135399103165, + "r2": 0.6752781867980957, + "score": -0.22544924405132602 + }, + "val": { + "rmse": 0.2334750859643239, + "mae": 0.12858723104000092, + "r2": 0.6358902454376221, + "score": -0.2334750859643239 + }, + "test": { + "rmse": 0.23518646749461922, + "mae": 0.15039512515068054, + "r2": 0.6220239400863647, + "score": -0.23518646749461922 + } + }, + "time": "0:01:27.894088", + "diversity": { + "train": { + "mean_centered_corr": -0.018592098727822304, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045505620539188385 + }, + "val": { + "mean_centered_corr": -0.018926210701465607, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06432383507490158 + }, + "test": { + "mean_centered_corr": -0.012847395613789558, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09277871996164322 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 6, + 19, + 25, + 21, + 29 + ], + "head_selection_time": "0:00:00.103487" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..138288004 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24049199255343762, + "mae": 0.12693814933300018, + "r2": 0.6304994225502014, + "score": -0.24049199255343762 + }, + "val": { + "rmse": 0.23716996532706527, + "mae": 0.13215987384319305, + "r2": 0.624274492263794, + "score": -0.23716996532706527 + }, + "test": { + "rmse": 0.23470593771148443, + "mae": 0.14771248400211334, + "r2": 0.6235669851303101, + "score": -0.23470593771148443 + } + }, + "time": "0:01:24.453355", + "diversity": { + "train": { + "mean_centered_corr": -0.0185525044798851, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045574937015771866 + }, + "val": { + "mean_centered_corr": -0.019020110368728638, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05756770819425583 + }, + "test": { + "mean_centered_corr": -0.017030077055096626, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08178099244832993 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015188" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0.toml new file mode 100644 index 000000000..daef5efee --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..c6ec0be46 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23121498010281438, + "mae": 0.12121064215898514, + "r2": 0.6584566831588745, + "score": -0.23121498010281438 + }, + "val": { + "rmse": 0.23523322965051088, + "mae": 0.12707281112670898, + "r2": 0.6303858757019043, + "score": -0.23523322965051088 + }, + "test": { + "rmse": 0.23438947156097584, + "mae": 0.14751991629600525, + "r2": 0.6245813965797424, + "score": -0.23438947156097584 + } + }, + "time": "0:01:24.453355", + "diversity": { + "train": { + "mean_centered_corr": -0.0185525044798851, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045574937015771866 + }, + "val": { + "mean_centered_corr": -0.019020110368728638, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05756770819425583 + }, + "test": { + "mean_centered_corr": -0.017030077055096626, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08178099244832993 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..daaa60081 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,662 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2346798186181653, + "mae": 0.12336737662553787, + "r2": 0.6481436491012573, + "score": -0.2346798186181653 + }, + "val": { + "rmse": 0.23379921122284117, + "mae": 0.12755374610424042, + "r2": 0.6348785161972046, + "score": -0.23379921122284117 + }, + "test": { + "rmse": 0.23898482806189927, + "mae": 0.15705350041389465, + "r2": 0.6097164154052734, + "score": -0.23898482806189927 + } + }, + "time": "0:01:24.453355", + "diversity": { + "train": { + "mean_centered_corr": -0.0185525044798851, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045574937015771866 + }, + "val": { + "mean_centered_corr": -0.019020110368728638, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05756770819425583 + }, + "test": { + "mean_centered_corr": -0.017030077055096626, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08178099244832993 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 25, + 21, + 0, + 1 + ], + "head_selection_time": "0:00:00.088168" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b8bf7c2d1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2368178141443934, + "mae": 0.12198712676763535, + "r2": 0.6417034864425659, + "score": -0.2368178141443934 + }, + "val": { + "rmse": 0.23907342929415865, + "mae": 0.13113287091255188, + "r2": 0.6182193756103516, + "score": -0.23907342929415865 + }, + "test": { + "rmse": 0.23310698183908637, + "mae": 0.14301016926765442, + "r2": 0.6286784410476685, + "score": -0.23310698183908637 + } + }, + "time": "0:01:21.540934", + "diversity": { + "train": { + "mean_centered_corr": -0.018635282292962074, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04599732533097267 + }, + "val": { + "mean_centered_corr": -0.01881239004433155, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06206817552447319 + }, + "test": { + "mean_centered_corr": -0.017686480656266212, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08751651644706726 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28 + ], + "head_selection_time": "0:00:00.015354" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0.toml new file mode 100644 index 000000000..c35d423aa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..8bf55312b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23194337523062297, + "mae": 0.12280548363924026, + "r2": 0.6563013792037964, + "score": -0.23194337523062297 + }, + "val": { + "rmse": 0.23519216180885402, + "mae": 0.1276979148387909, + "r2": 0.6305148601531982, + "score": -0.23519216180885402 + }, + "test": { + "rmse": 0.23499174827687583, + "mae": 0.1487179696559906, + "r2": 0.622649610042572, + "score": -0.23499174827687583 + } + }, + "time": "0:01:21.540934", + "diversity": { + "train": { + "mean_centered_corr": -0.018635282292962074, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04599732533097267 + }, + "val": { + "mean_centered_corr": -0.01881239004433155, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06206817552447319 + }, + "test": { + "mean_centered_corr": -0.017686480656266212, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08751651644706726 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..877fe50a7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,667 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23278222079157185, + "mae": 0.12435509264469147, + "r2": 0.6538107991218567, + "score": -0.23278222079157185 + }, + "val": { + "rmse": 0.2332399859921054, + "mae": 0.12662732601165771, + "r2": 0.6366231441497803, + "score": -0.2332399859921054 + }, + "test": { + "rmse": 0.2326072306387775, + "mae": 0.14432717859745026, + "r2": 0.6302688717842102, + "score": -0.2326072306387775 + } + }, + "time": "0:01:21.540934", + "diversity": { + "train": { + "mean_centered_corr": -0.018635282292962074, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04599732533097267 + }, + "val": { + "mean_centered_corr": -0.01881239004433155, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06206817552447319 + }, + "test": { + "mean_centered_corr": -0.017686480656266212, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08751651644706726 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28, + 25, + 29, + 21, + 0, + 19, + 20, + 25, + 1, + 6 + ], + "head_selection_time": "0:00:00.170893" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bfbb3aa87 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23711698667572978, + "mae": 0.1269228607416153, + "r2": 0.6407976150512695, + "score": -0.23711698667572978 + }, + "val": { + "rmse": 0.2376162677316268, + "mae": 0.13255804777145386, + "r2": 0.6228591203689575, + "score": -0.2376162677316268 + }, + "test": { + "rmse": 0.23517437354931303, + "mae": 0.14232948422431946, + "r2": 0.6220628023147583, + "score": -0.23517437354931303 + } + }, + "time": "0:01:21.799996", + "diversity": { + "train": { + "mean_centered_corr": -0.01871204562485218, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04567575082182884 + }, + "val": { + "mean_centered_corr": -0.01849065162241459, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058917272835969925 + }, + "test": { + "mean_centered_corr": -0.016842441633343697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08555692434310913 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.017414" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0.toml new file mode 100644 index 000000000..60de6d162 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..71b8eb184 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23218298905099302, + "mae": 0.12276018410921097, + "r2": 0.6555908918380737, + "score": -0.23218298905099302 + }, + "val": { + "rmse": 0.23534793771825302, + "mae": 0.12849925458431244, + "r2": 0.6300252676010132, + "score": -0.23534793771825302 + }, + "test": { + "rmse": 0.2358765820307155, + "mae": 0.1505100429058075, + "r2": 0.6198024749755859, + "score": -0.2358765820307155 + } + }, + "time": "0:01:21.799996", + "diversity": { + "train": { + "mean_centered_corr": -0.01871204562485218, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04567575082182884 + }, + "val": { + "mean_centered_corr": -0.01849065162241459, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058917272835969925 + }, + "test": { + "mean_centered_corr": -0.016842441633343697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08555692434310913 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ea4c5da7c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,665 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23342471593298392, + "mae": 0.1249859556555748, + "r2": 0.6518971920013428, + "score": -0.23342471593298392 + }, + "val": { + "rmse": 0.2335073783441898, + "mae": 0.12728865444660187, + "r2": 0.6357895135879517, + "score": -0.2335073783441898 + }, + "test": { + "rmse": 0.2329070852784964, + "mae": 0.1443146914243698, + "r2": 0.6293150186538696, + "score": -0.2329070852784964 + } + }, + "time": "0:01:21.799996", + "diversity": { + "train": { + "mean_centered_corr": -0.01871204562485218, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04567575082182884 + }, + "val": { + "mean_centered_corr": -0.01849065162241459, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058917272835969925 + }, + "test": { + "mean_centered_corr": -0.016842441633343697, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08555692434310913 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29, + 0, + 28, + 19, + 25, + 13, + 21, + 0 + ], + "head_selection_time": "0:00:00.138920" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8b6bd1cf1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.234490000803403, + "mae": 0.12728147208690643, + "r2": 0.6487126350402832, + "score": -0.234490000803403 + }, + "val": { + "rmse": 0.23720560242248867, + "mae": 0.1349465399980545, + "r2": 0.6241616010665894, + "score": -0.23720560242248867 + }, + "test": { + "rmse": 0.23787240614757268, + "mae": 0.1500697135925293, + "r2": 0.6133413314819336, + "score": -0.23787240614757268 + } + }, + "time": "0:01:25.321284", + "diversity": { + "train": { + "mean_centered_corr": -0.018023740500211716, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047287896275520325 + }, + "val": { + "mean_centered_corr": -0.01780591532588005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06861133128404617 + }, + "test": { + "mean_centered_corr": -0.015070421621203423, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09827303886413574 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.015329" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0.toml new file mode 100644 index 000000000..5bb2a3fa1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..9a822931b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2240611226745793, + "mae": 0.11743048578500748, + "r2": 0.6792646050453186, + "score": -0.2240611226745793 + }, + "val": { + "rmse": 0.23499558463666298, + "mae": 0.12714111804962158, + "r2": 0.6311322450637817, + "score": -0.23499558463666298 + }, + "test": { + "rmse": 0.23934769790965568, + "mae": 0.15463049709796906, + "r2": 0.6085303425788879, + "score": -0.23934769790965568 + } + }, + "time": "0:01:25.321284", + "diversity": { + "train": { + "mean_centered_corr": -0.018023740500211716, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047287896275520325 + }, + "val": { + "mean_centered_corr": -0.01780591532588005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06861133128404617 + }, + "test": { + "mean_centered_corr": -0.015070421621203423, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09827303886413574 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d0b6cd254 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r2_n1em05_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,665 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22719550493821902, + "mae": 0.11786691844463348, + "r2": 0.6702283024787903, + "score": -0.22719550493821902 + }, + "val": { + "rmse": 0.23350963577191214, + "mae": 0.1265878677368164, + "r2": 0.6357824802398682, + "score": -0.23350963577191214 + }, + "test": { + "rmse": 0.2393402580368093, + "mae": 0.15526840090751648, + "r2": 0.6085547208786011, + "score": -0.2393402580368093 + } + }, + "time": "0:01:25.321284", + "diversity": { + "train": { + "mean_centered_corr": -0.018023740500211716, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047287896275520325 + }, + "val": { + "mean_centered_corr": -0.01780591532588005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06861133128404617 + }, + "test": { + "mean_centered_corr": -0.015070421621203423, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09827303886413574 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29, + 10, + 7, + 22, + 4, + 8, + 25, + 21 + ], + "head_selection_time": "0:00:00.130729" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bafb9519d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22820973096491176, + "mae": 0.12737685441970825, + "r2": 0.667277455329895, + "score": -0.22820973096491176 + }, + "val": { + "rmse": 0.23905700508983188, + "mae": 0.14095449447631836, + "r2": 0.6182718276977539, + "score": -0.23905700508983188 + }, + "test": { + "rmse": 0.2554892709638668, + "mae": 0.17996472120285034, + "r2": 0.5539485812187195, + "score": -0.2554892709638668 + } + }, + "time": "0:02:00.734464", + "diversity": { + "train": { + "mean_centered_corr": -0.019317513331770897, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04794894531369209 + }, + "val": { + "mean_centered_corr": -0.019240468740463257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.065296970307827 + }, + "test": { + "mean_centered_corr": -0.018747208639979362, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741389214992523 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.016010" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0.toml new file mode 100644 index 000000000..29d629616 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..eba302088 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22218533123207995, + "mae": 0.11801747977733612, + "r2": 0.6846123933792114, + "score": -0.22218533123207995 + }, + "val": { + "rmse": 0.23516964509902982, + "mae": 0.1285666972398758, + "r2": 0.6305856704711914, + "score": -0.23516964509902982 + }, + "test": { + "rmse": 0.23928895864366434, + "mae": 0.15676262974739075, + "r2": 0.608722448348999, + "score": -0.23928895864366434 + } + }, + "time": "0:02:00.734464", + "diversity": { + "train": { + "mean_centered_corr": -0.019317513331770897, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04794894531369209 + }, + "val": { + "mean_centered_corr": -0.019240468740463257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.065296970307827 + }, + "test": { + "mean_centered_corr": -0.018747208639979362, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741389214992523 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ce12b44c5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,664 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22489126180626096, + "mae": 0.11878597736358643, + "r2": 0.6768835783004761, + "score": -0.22489126180626096 + }, + "val": { + "rmse": 0.23363252496566134, + "mae": 0.12608498334884644, + "r2": 0.6353989839553833, + "score": -0.23363252496566134 + }, + "test": { + "rmse": 0.2362188764537236, + "mae": 0.15149326622486115, + "r2": 0.618698239326477, + "score": -0.2362188764537236 + } + }, + "time": "0:02:00.734464", + "diversity": { + "train": { + "mean_centered_corr": -0.019317513331770897, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04794894531369209 + }, + "val": { + "mean_centered_corr": -0.019240468740463257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.065296970307827 + }, + "test": { + "mean_centered_corr": -0.018747208639979362, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08741389214992523 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 8, + 22, + 7, + 29, + 25, + 21 + ], + "head_selection_time": "0:00:00.119887" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..df545e284 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23007882803014523, + "mae": 0.12374971061944962, + "r2": 0.6618049740791321, + "score": -0.23007882803014523 + }, + "val": { + "rmse": 0.23901989843968774, + "mae": 0.13069558143615723, + "r2": 0.6183903217315674, + "score": -0.23901989843968774 + }, + "test": { + "rmse": 0.24063495640158664, + "mae": 0.1513846516609192, + "r2": 0.6043082475662231, + "score": -0.24063495640158664 + } + }, + "time": "0:02:00.389497", + "diversity": { + "train": { + "mean_centered_corr": -0.018541699275374413, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04868823289871216 + }, + "val": { + "mean_centered_corr": -0.019729774445295334, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561318784952164 + }, + "test": { + "mean_centered_corr": -0.01752825267612934, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573257714509964 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015353" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0.toml new file mode 100644 index 000000000..713099f48 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..2a8f194cc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22270596936228843, + "mae": 0.11821265518665314, + "r2": 0.6831325888633728, + "score": -0.22270596936228843 + }, + "val": { + "rmse": 0.23541076200529618, + "mae": 0.1285439282655716, + "r2": 0.6298277378082275, + "score": -0.23541076200529618 + }, + "test": { + "rmse": 0.2395584319829453, + "mae": 0.15599201619625092, + "r2": 0.6078406572341919, + "score": -0.2395584319829453 + } + }, + "time": "0:02:00.389497", + "diversity": { + "train": { + "mean_centered_corr": -0.018541699275374413, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04868823289871216 + }, + "val": { + "mean_centered_corr": -0.019729774445295334, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561318784952164 + }, + "test": { + "mean_centered_corr": -0.01752825267612934, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573257714509964 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..bd9e463d1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,663 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22560705771393183, + "mae": 0.11890538036823273, + "r2": 0.6748234033584595, + "score": -0.22560705771393183 + }, + "val": { + "rmse": 0.23373660723675202, + "mae": 0.1272813230752945, + "r2": 0.6350740790367126, + "score": -0.23373660723675202 + }, + "test": { + "rmse": 0.23443243594536417, + "mae": 0.14499256014823914, + "r2": 0.624443769454956, + "score": -0.23443243594536417 + } + }, + "time": "0:02:00.389497", + "diversity": { + "train": { + "mean_centered_corr": -0.018541699275374413, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04868823289871216 + }, + "val": { + "mean_centered_corr": -0.019729774445295334, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06561318784952164 + }, + "test": { + "mean_centered_corr": -0.01752825267612934, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09573257714509964 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 29, + 8, + 0, + 19, + 22 + ], + "head_selection_time": "0:00:00.101274" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3f5971d21 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23015652536762907, + "mae": 0.12545116245746613, + "r2": 0.661576509475708, + "score": -0.23015652536762907 + }, + "val": { + "rmse": 0.239085684381371, + "mae": 0.13462616503238678, + "r2": 0.6181802749633789, + "score": -0.239085684381371 + }, + "test": { + "rmse": 0.24513387433279307, + "mae": 0.16250139474868774, + "r2": 0.5893741846084595, + "score": -0.24513387433279307 + } + }, + "time": "0:01:58.386465", + "diversity": { + "train": { + "mean_centered_corr": -0.01839604787528515, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0476016141474247 + }, + "val": { + "mean_centered_corr": -0.0184602253139019, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696488708257675 + }, + "test": { + "mean_centered_corr": -0.018153522163629532, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09845950454473495 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.014850" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0.toml new file mode 100644 index 000000000..2e5997daa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..f9eec6cd8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22416379102176556, + "mae": 0.11926887929439545, + "r2": 0.6789705753326416, + "score": -0.22416379102176556 + }, + "val": { + "rmse": 0.23516104335271928, + "mae": 0.12952451407909393, + "r2": 0.6306126713752747, + "score": -0.23516104335271928 + }, + "test": { + "rmse": 0.24165658285763553, + "mae": 0.1609659045934677, + "r2": 0.6009412407875061, + "score": -0.24165658285763553 + } + }, + "time": "0:01:58.386465", + "diversity": { + "train": { + "mean_centered_corr": -0.01839604787528515, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0476016141474247 + }, + "val": { + "mean_centered_corr": -0.0184602253139019, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696488708257675 + }, + "test": { + "mean_centered_corr": -0.018153522163629532, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09845950454473495 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..46cbec9f2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,667 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22445475190340167, + "mae": 0.11813689768314362, + "r2": 0.6781366467475891, + "score": -0.22445475190340167 + }, + "val": { + "rmse": 0.23356035433981295, + "mae": 0.1266433298587799, + "r2": 0.6356241703033447, + "score": -0.23356035433981295 + }, + "test": { + "rmse": 0.2372102431750833, + "mae": 0.15235204994678497, + "r2": 0.6154910326004028, + "score": -0.2372102431750833 + } + }, + "time": "0:01:58.386465", + "diversity": { + "train": { + "mean_centered_corr": -0.01839604787528515, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0476016141474247 + }, + "val": { + "mean_centered_corr": -0.0184602253139019, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06696488708257675 + }, + "test": { + "mean_centered_corr": -0.018153522163629532, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09845950454473495 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 7, + 4, + 22, + 19, + 8, + 18, + 7, + 29, + 25 + ], + "head_selection_time": "0:00:00.157297" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c50fb3660 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23893329644403358, + "mae": 0.12647750973701477, + "r2": 0.6352735757827759, + "score": -0.23893329644403358 + }, + "val": { + "rmse": 0.23785555444789067, + "mae": 0.1310475766658783, + "r2": 0.6220991611480713, + "score": -0.23785555444789067 + }, + "test": { + "rmse": 0.23748897727684737, + "mae": 0.1446538269519806, + "r2": 0.6145868301391602, + "score": -0.23748897727684737 + } + }, + "time": "0:01:52.701352", + "diversity": { + "train": { + "mean_centered_corr": -0.019367322325706482, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785257205367088 + }, + "val": { + "mean_centered_corr": -0.0196691807359457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109632924199104 + }, + "test": { + "mean_centered_corr": -0.01985469087958336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08116098493337631 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.016291" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0.toml new file mode 100644 index 000000000..2cb767b7b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..4a4d01205 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2325909984973762, + "mae": 0.12197329849004745, + "r2": 0.6543793678283691, + "score": -0.2325909984973762 + }, + "val": { + "rmse": 0.23537506682607195, + "mae": 0.12669186294078827, + "r2": 0.6299399733543396, + "score": -0.23537506682607195 + }, + "test": { + "rmse": 0.23579029486809283, + "mae": 0.14991408586502075, + "r2": 0.6200805902481079, + "score": -0.23579029486809283 + } + }, + "time": "0:01:52.701352", + "diversity": { + "train": { + "mean_centered_corr": -0.019367322325706482, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785257205367088 + }, + "val": { + "mean_centered_corr": -0.0196691807359457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109632924199104 + }, + "test": { + "mean_centered_corr": -0.01985469087958336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08116098493337631 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..65532463f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,667 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23476379254176588, + "mae": 0.12291711568832397, + "r2": 0.6478918194770813, + "score": -0.23476379254176588 + }, + "val": { + "rmse": 0.23340542834677785, + "mae": 0.12670302391052246, + "r2": 0.6361074447631836, + "score": -0.23340542834677785 + }, + "test": { + "rmse": 0.23829137280333715, + "mae": 0.15507613122463226, + "r2": 0.6119781136512756, + "score": -0.23829137280333715 + } + }, + "time": "0:01:52.701352", + "diversity": { + "train": { + "mean_centered_corr": -0.019367322325706482, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04785257205367088 + }, + "val": { + "mean_centered_corr": -0.0196691807359457, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06109632924199104 + }, + "test": { + "mean_centered_corr": -0.01985469087958336, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08116098493337631 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29, + 25, + 28, + 0, + 16, + 21, + 4, + 25, + 1, + 6 + ], + "head_selection_time": "0:00:00.166251" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..530b524b7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22528267991110515, + "mae": 0.12274349480867386, + "r2": 0.6757578253746033, + "score": -0.22528267991110515 + }, + "val": { + "rmse": 0.23779726122481318, + "mae": 0.13489505648612976, + "r2": 0.6222844123840332, + "score": -0.23779726122481318 + }, + "test": { + "rmse": 0.24153146027417532, + "mae": 0.15904922783374786, + "r2": 0.6013543605804443, + "score": -0.24153146027417532 + } + }, + "time": "0:01:59.320916", + "diversity": { + "train": { + "mean_centered_corr": -0.018156183883547783, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047758832573890686 + }, + "val": { + "mean_centered_corr": -0.01822330430150032, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06243854761123657 + }, + "test": { + "mean_centered_corr": -0.015858400613069534, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08906728774309158 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.015216" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0.toml new file mode 100644 index 000000000..bc810cded --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..ce31664a1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22031523790754975, + "mae": 0.11761360615491867, + "r2": 0.6898991465568542, + "score": -0.22031523790754975 + }, + "val": { + "rmse": 0.2350763793606662, + "mae": 0.12925966084003448, + "r2": 0.6308785676956177, + "score": -0.2350763793606662 + }, + "test": { + "rmse": 0.23832022238784234, + "mae": 0.15449607372283936, + "r2": 0.6118841171264648, + "score": -0.23832022238784234 + } + }, + "time": "0:01:59.320916", + "diversity": { + "train": { + "mean_centered_corr": -0.018156183883547783, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047758832573890686 + }, + "val": { + "mean_centered_corr": -0.01822330430150032, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06243854761123657 + }, + "test": { + "mean_centered_corr": -0.015858400613069534, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08906728774309158 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..74a0303ab --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,664 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2219229132672244, + "mae": 0.11750265210866928, + "r2": 0.6853569746017456, + "score": -0.2219229132672244 + }, + "val": { + "rmse": 0.23334639858951206, + "mae": 0.12622985243797302, + "r2": 0.63629150390625, + "score": -0.23334639858951206 + }, + "test": { + "rmse": 0.23179329091828219, + "mae": 0.1403104066848755, + "r2": 0.6328518390655518, + "score": -0.23179329091828219 + } + }, + "time": "0:01:59.320916", + "diversity": { + "train": { + "mean_centered_corr": -0.018156183883547783, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047758832573890686 + }, + "val": { + "mean_centered_corr": -0.01822330430150032, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06243854761123657 + }, + "test": { + "mean_centered_corr": -0.015858400613069534, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08906728774309158 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 7, + 29, + 25, + 19, + 8, + 31 + ], + "head_selection_time": "0:00:00.115448" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..92cc514d0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2386679091803034, + "mae": 0.13499177992343903, + "r2": 0.6360833644866943, + "score": -0.2386679091803034 + }, + "val": { + "rmse": 0.23814119850842008, + "mae": 0.13966409862041473, + "r2": 0.6211909651756287, + "score": -0.23814119850842008 + }, + "test": { + "rmse": 0.2663355327951239, + "mae": 0.1914888322353363, + "r2": 0.5152723789215088, + "score": -0.2663355327951239 + } + }, + "time": "0:01:52.189349", + "diversity": { + "train": { + "mean_centered_corr": -0.019499018788337708, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044644709676504135 + }, + "val": { + "mean_centered_corr": -0.019907629117369652, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05747465789318085 + }, + "test": { + "mean_centered_corr": -0.016552036628127098, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08085940778255463 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015272" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0.toml new file mode 100644 index 000000000..c530ba33a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..1d7b35844 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23138006715156628, + "mae": 0.12110890448093414, + "r2": 0.6579687595367432, + "score": -0.23138006715156628 + }, + "val": { + "rmse": 0.2350520527289723, + "mae": 0.12721683084964752, + "r2": 0.6309549808502197, + "score": -0.2350520527289723 + }, + "test": { + "rmse": 0.23495368248701565, + "mae": 0.14864283800125122, + "r2": 0.6227718591690063, + "score": -0.23495368248701565 + } + }, + "time": "0:01:52.189349", + "diversity": { + "train": { + "mean_centered_corr": -0.019499018788337708, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044644709676504135 + }, + "val": { + "mean_centered_corr": -0.019907629117369652, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05747465789318085 + }, + "test": { + "mean_centered_corr": -0.016552036628127098, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08085940778255463 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1896aec73 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,663 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2324669097097326, + "mae": 0.12446025013923645, + "r2": 0.6547480225563049, + "score": -0.2324669097097326 + }, + "val": { + "rmse": 0.23343416363529057, + "mae": 0.12728102505207062, + "r2": 0.6360177993774414, + "score": -0.23343416363529057 + }, + "test": { + "rmse": 0.234433762813203, + "mae": 0.14741288125514984, + "r2": 0.6244394779205322, + "score": -0.234433762813203 + } + }, + "time": "0:01:52.189349", + "diversity": { + "train": { + "mean_centered_corr": -0.019499018788337708, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044644709676504135 + }, + "val": { + "mean_centered_corr": -0.019907629117369652, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05747465789318085 + }, + "test": { + "mean_centered_corr": -0.016552036628127098, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08085940778255463 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 21, + 0, + 4, + 6, + 19 + ], + "head_selection_time": "0:00:00.103853" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3e1dabc20 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23445258445893527, + "mae": 0.12787146866321564, + "r2": 0.6488246917724609, + "score": -0.23445258445893527 + }, + "val": { + "rmse": 0.23744052562658574, + "mae": 0.1366073340177536, + "r2": 0.6234167814254761, + "score": -0.23744052562658574 + }, + "test": { + "rmse": 0.2382342886377602, + "mae": 0.15353862941265106, + "r2": 0.6121640205383301, + "score": -0.2382342886377602 + } + }, + "time": "0:01:52.520601", + "diversity": { + "train": { + "mean_centered_corr": -0.019367916509509087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04525168240070343 + }, + "val": { + "mean_centered_corr": -0.019405901432037354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05762277543544769 + }, + "test": { + "mean_centered_corr": -0.018559793010354042, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07860848307609558 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:00.015170" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0.toml new file mode 100644 index 000000000..4549a6ec0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..d9d852c8d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23102999093285262, + "mae": 0.1207849308848381, + "r2": 0.659002959728241, + "score": -0.23102999093285262 + }, + "val": { + "rmse": 0.23466412671675002, + "mae": 0.12721090018749237, + "r2": 0.6321721076965332, + "score": -0.23466412671675002 + }, + "test": { + "rmse": 0.2324620941369229, + "mae": 0.14480625092983246, + "r2": 0.630730152130127, + "score": -0.2324620941369229 + } + }, + "time": "0:01:52.520601", + "diversity": { + "train": { + "mean_centered_corr": -0.019367916509509087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04525168240070343 + }, + "val": { + "mean_centered_corr": -0.019405901432037354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05762277543544769 + }, + "test": { + "mean_centered_corr": -0.018559793010354042, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07860848307609558 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1ea4d8c63 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,673 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23228452935964564, + "mae": 0.12128094583749771, + "r2": 0.6552895307540894, + "score": -0.23228452935964564 + }, + "val": { + "rmse": 0.23321172995482362, + "mae": 0.12664632499217987, + "r2": 0.6367111802101135, + "score": -0.23321172995482362 + }, + "test": { + "rmse": 0.234184776040884, + "mae": 0.1486988514661789, + "r2": 0.6252368688583374, + "score": -0.234184776040884 + } + }, + "time": "0:01:52.520601", + "diversity": { + "train": { + "mean_centered_corr": -0.019367916509509087, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04525168240070343 + }, + "val": { + "mean_centered_corr": -0.019405901432037354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05762277543544769 + }, + "test": { + "mean_centered_corr": -0.018559793010354042, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07860848307609558 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 21, + 25, + 29, + 10, + 4, + 25, + 15, + 0, + 19, + 21, + 25, + 29, + 28, + 25, + 21 + ], + "head_selection_time": "0:00:00.249510" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bf2abe48d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23954623995992808, + "mae": 0.12636718153953552, + "r2": 0.6333998441696167, + "score": -0.23954623995992808 + }, + "val": { + "rmse": 0.23869658059774151, + "mae": 0.1339649260044098, + "r2": 0.6194220185279846, + "score": -0.23869658059774151 + }, + "test": { + "rmse": 0.24052100411353006, + "mae": 0.1552955061197281, + "r2": 0.6046829223632812, + "score": -0.24052100411353006 + } + }, + "time": "0:01:52.324536", + "diversity": { + "train": { + "mean_centered_corr": -0.018679402768611908, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04577600955963135 + }, + "val": { + "mean_centered_corr": -0.019475068897008896, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05786792188882828 + }, + "test": { + "mean_centered_corr": -0.017192553728818893, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07846961915493011 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015515" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0.toml new file mode 100644 index 000000000..c4040812f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..29506ab1d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2307699869541623, + "mae": 0.12244833260774612, + "r2": 0.6597700119018555, + "score": -0.2307699869541623 + }, + "val": { + "rmse": 0.23532386870106714, + "mae": 0.12836803495883942, + "r2": 0.6301009654998779, + "score": -0.23532386870106714 + }, + "test": { + "rmse": 0.23728542485175996, + "mae": 0.1531296819448471, + "r2": 0.6152472496032715, + "score": -0.23728542485175996 + } + }, + "time": "0:01:52.324536", + "diversity": { + "train": { + "mean_centered_corr": -0.018679402768611908, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04577600955963135 + }, + "val": { + "mean_centered_corr": -0.019475068897008896, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05786792188882828 + }, + "test": { + "mean_centered_corr": -0.017192553728818893, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07846961915493011 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..adff072f4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p0001_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,668 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23278514938222616, + "mae": 0.12298233807086945, + "r2": 0.6538020968437195, + "score": -0.23278514938222616 + }, + "val": { + "rmse": 0.23353358071122673, + "mae": 0.1262781023979187, + "r2": 0.6357077360153198, + "score": -0.23353358071122673 + }, + "test": { + "rmse": 0.23738632838570875, + "mae": 0.15430940687656403, + "r2": 0.6149200201034546, + "score": -0.23738632838570875 + } + }, + "time": "0:01:52.324536", + "diversity": { + "train": { + "mean_centered_corr": -0.018679402768611908, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04577600955963135 + }, + "val": { + "mean_centered_corr": -0.019475068897008896, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05786792188882828 + }, + "test": { + "mean_centered_corr": -0.017192553728818893, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07846961915493011 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 21, + 25, + 29, + 28, + 0, + 21, + 1, + 25, + 21, + 0 + ], + "head_selection_time": "0:00:00.177379" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..31f233e53 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2247163600717624, + "mae": 0.12514503300189972, + "r2": 0.6773859262466431, + "score": -0.2247163600717624 + }, + "val": { + "rmse": 0.23828438459742213, + "mae": 0.13836915791034698, + "r2": 0.6207352876663208, + "score": -0.23828438459742213 + }, + "test": { + "rmse": 0.2530485190714714, + "mae": 0.17723064124584198, + "r2": 0.5624303817749023, + "score": -0.2530485190714714 + } + }, + "time": "0:02:02.636955", + "diversity": { + "train": { + "mean_centered_corr": -0.018269460648298264, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04634244740009308 + }, + "val": { + "mean_centered_corr": -0.018611595034599304, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06517667323350906 + }, + "test": { + "mean_centered_corr": -0.01696156896650791, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09637612849473953 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015773" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0.toml new file mode 100644 index 000000000..c0b709eb7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..5f27feefe --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.21980748699677205, + "mae": 0.11748126894235611, + "r2": 0.6913268566131592, + "score": -0.21980748699677205 + }, + "val": { + "rmse": 0.2349816418639806, + "mae": 0.12830044329166412, + "r2": 0.6311760544776917, + "score": -0.2349816418639806 + }, + "test": { + "rmse": 0.24136844697035015, + "mae": 0.15835422277450562, + "r2": 0.6018922924995422, + "score": -0.24136844697035015 + } + }, + "time": "0:02:02.636955", + "diversity": { + "train": { + "mean_centered_corr": -0.018269460648298264, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04634244740009308 + }, + "val": { + "mean_centered_corr": -0.018611595034599304, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06517667323350906 + }, + "test": { + "mean_centered_corr": -0.01696156896650791, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09637612849473953 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..719e2a382 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,665 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22116132451653123, + "mae": 0.11670376360416412, + "r2": 0.6875127553939819, + "score": -0.22116132451653123 + }, + "val": { + "rmse": 0.23357213312617114, + "mae": 0.1273292750120163, + "r2": 0.6355874538421631, + "score": -0.23357213312617114 + }, + "test": { + "rmse": 0.23844724876452805, + "mae": 0.15294302999973297, + "r2": 0.6114702820777893, + "score": -0.23844724876452805 + } + }, + "time": "0:02:02.636955", + "diversity": { + "train": { + "mean_centered_corr": -0.018269460648298264, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04634244740009308 + }, + "val": { + "mean_centered_corr": -0.018611595034599304, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06517667323350906 + }, + "test": { + "mean_centered_corr": -0.01696156896650791, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09637612849473953 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 21, + 29, + 7, + 10, + 8, + 19, + 18 + ], + "head_selection_time": "0:00:00.134808" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c663085b9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23150610628324156, + "mae": 0.12310612946748734, + "r2": 0.6575959920883179, + "score": -0.23150610628324156 + }, + "val": { + "rmse": 0.239770922958097, + "mae": 0.1303773671388626, + "r2": 0.6159884333610535, + "score": -0.239770922958097 + }, + "test": { + "rmse": 0.23915230117801708, + "mae": 0.14451579749584198, + "r2": 0.6091692447662354, + "score": -0.23915230117801708 + } + }, + "time": "0:01:59.760644", + "diversity": { + "train": { + "mean_centered_corr": -0.018444906920194626, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04715638607740402 + }, + "val": { + "mean_centered_corr": -0.01882033608853817, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07111956179141998 + }, + "test": { + "mean_centered_corr": -0.016320303082466125, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09910114109516144 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.015389" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0.toml new file mode 100644 index 000000000..932eafb54 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..f524ca666 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22098960681519272, + "mae": 0.11940933763980865, + "r2": 0.6879978179931641, + "score": -0.22098960681519272 + }, + "val": { + "rmse": 0.23549047304076126, + "mae": 0.12960220873355865, + "r2": 0.6295770406723022, + "score": -0.23549047304076126 + }, + "test": { + "rmse": 0.24128326690664834, + "mae": 0.15859289467334747, + "r2": 0.6021732091903687, + "score": -0.24128326690664834 + } + }, + "time": "0:01:59.760644", + "diversity": { + "train": { + "mean_centered_corr": -0.018444906920194626, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04715638607740402 + }, + "val": { + "mean_centered_corr": -0.01882033608853817, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07111956179141998 + }, + "test": { + "mean_centered_corr": -0.016320303082466125, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09910114109516144 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5fc0906bc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,666 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22227666522397582, + "mae": 0.11813152581453323, + "r2": 0.6843529939651489, + "score": -0.22227666522397582 + }, + "val": { + "rmse": 0.23367947054972119, + "mae": 0.12708741426467896, + "r2": 0.6352524757385254, + "score": -0.23367947054972119 + }, + "test": { + "rmse": 0.23861273393383206, + "mae": 0.15416637063026428, + "r2": 0.6109308004379272, + "score": -0.23861273393383206 + } + }, + "time": "0:01:59.760644", + "diversity": { + "train": { + "mean_centered_corr": -0.018444906920194626, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04715638607740402 + }, + "val": { + "mean_centered_corr": -0.01882033608853817, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07111956179141998 + }, + "test": { + "mean_centered_corr": -0.016320303082466125, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09910114109516144 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8, + 4, + 21, + 25, + 19, + 22, + 7, + 10, + 27 + ], + "head_selection_time": "0:00:00.144065" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9e6107da0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23737684181165758, + "mae": 0.12491133064031601, + "r2": 0.640009880065918, + "score": -0.23737684181165758 + }, + "val": { + "rmse": 0.23850314174188256, + "mae": 0.13284452259540558, + "r2": 0.620038628578186, + "score": -0.23850314174188256 + }, + "test": { + "rmse": 0.23244720609452174, + "mae": 0.14553889632225037, + "r2": 0.6307774186134338, + "score": -0.23244720609452174 + } + }, + "time": "0:01:51.542856", + "diversity": { + "train": { + "mean_centered_corr": -0.018743393942713737, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045543067157268524 + }, + "val": { + "mean_centered_corr": -0.01886933483183384, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060780204832553864 + }, + "test": { + "mean_centered_corr": -0.017534511163830757, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08134183287620544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28 + ], + "head_selection_time": "0:00:00.015619" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0.toml new file mode 100644 index 000000000..45c54b15b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..6eb74283d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23111020547391228, + "mae": 0.12324945628643036, + "r2": 0.6587661504745483, + "score": -0.23111020547391228 + }, + "val": { + "rmse": 0.23543217967973734, + "mae": 0.1291837990283966, + "r2": 0.6297603845596313, + "score": -0.23543217967973734 + }, + "test": { + "rmse": 0.23786624351296862, + "mae": 0.1546480357646942, + "r2": 0.6133613586425781, + "score": -0.23786624351296862 + } + }, + "time": "0:01:51.542856", + "diversity": { + "train": { + "mean_centered_corr": -0.018743393942713737, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045543067157268524 + }, + "val": { + "mean_centered_corr": -0.01886933483183384, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060780204832553864 + }, + "test": { + "mean_centered_corr": -0.017534511163830757, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08134183287620544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0aea97b14 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,665 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23300592784647, + "mae": 0.12180998921394348, + "r2": 0.6531450748443604, + "score": -0.23300592784647 + }, + "val": { + "rmse": 0.233651251716616, + "mae": 0.12775851786136627, + "r2": 0.6353405714035034, + "score": -0.233651251716616 + }, + "test": { + "rmse": 0.2427291888102559, + "mae": 0.16294866800308228, + "r2": 0.59739089012146, + "score": -0.2427291888102559 + } + }, + "time": "0:01:51.542856", + "diversity": { + "train": { + "mean_centered_corr": -0.018743393942713737, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045543067157268524 + }, + "val": { + "mean_centered_corr": -0.01886933483183384, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060780204832553864 + }, + "test": { + "mean_centered_corr": -0.017534511163830757, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08134183287620544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28, + 21, + 25, + 6, + 1, + 4, + 25, + 10 + ], + "head_selection_time": "0:00:00.134347" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..41d7ee2c9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2396765489575744, + "mae": 0.126370370388031, + "r2": 0.633000910282135, + "score": -0.2396765489575744 + }, + "val": { + "rmse": 0.23843109390982442, + "mae": 0.13302145898342133, + "r2": 0.6202681064605713, + "score": -0.23843109390982442 + }, + "test": { + "rmse": 0.24318669684901736, + "mae": 0.16076454520225525, + "r2": 0.5958717465400696, + "score": -0.24318669684901736 + } + }, + "time": "0:01:50.591643", + "diversity": { + "train": { + "mean_centered_corr": -0.019360007718205452, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04535945504903793 + }, + "val": { + "mean_centered_corr": -0.01878437213599682, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05948599800467491 + }, + "test": { + "mean_centered_corr": -0.020098138600587845, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08049041777849197 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015508" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0.toml new file mode 100644 index 000000000..9be18618b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..84fa26804 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2312604753596052, + "mae": 0.12086156755685806, + "r2": 0.6583222150802612, + "score": -0.2312604753596052 + }, + "val": { + "rmse": 0.23514095554484465, + "mae": 0.12670177221298218, + "r2": 0.6306757926940918, + "score": -0.23514095554484465 + }, + "test": { + "rmse": 0.23359667766611852, + "mae": 0.14610755443572998, + "r2": 0.6271167397499084, + "score": -0.23359667766611852 + } + }, + "time": "0:01:50.591643", + "diversity": { + "train": { + "mean_centered_corr": -0.019360007718205452, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04535945504903793 + }, + "val": { + "mean_centered_corr": -0.01878437213599682, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05948599800467491 + }, + "test": { + "mean_centered_corr": -0.020098138600587845, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08049041777849197 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..6c26dc9f8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n0p001_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,666 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23246381685745798, + "mae": 0.12184411287307739, + "r2": 0.6547572016716003, + "score": -0.23246381685745798 + }, + "val": { + "rmse": 0.2334736499346226, + "mae": 0.12559877336025238, + "r2": 0.6358946561813354, + "score": -0.2334736499346226 + }, + "test": { + "rmse": 0.23433651608184083, + "mae": 0.14739863574504852, + "r2": 0.6247509717941284, + "score": -0.23433651608184083 + } + }, + "time": "0:01:50.591643", + "diversity": { + "train": { + "mean_centered_corr": -0.019360007718205452, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04535945504903793 + }, + "val": { + "mean_centered_corr": -0.01878437213599682, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05948599800467491 + }, + "test": { + "mean_centered_corr": -0.020098138600587845, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08049041777849197 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 21, + 29, + 25, + 28, + 6, + 1, + 19 + ], + "head_selection_time": "0:00:00.146214" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1725f1d2c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22576688139974624, + "mae": 0.12466424703598022, + "r2": 0.6743625402450562, + "score": -0.22576688139974624 + }, + "val": { + "rmse": 0.2388823307131178, + "mae": 0.1381048709154129, + "r2": 0.6188294887542725, + "score": -0.2388823307131178 + }, + "test": { + "rmse": 0.25624351202497053, + "mae": 0.18258099257946014, + "r2": 0.5513110756874084, + "score": -0.25624351202497053 + } + }, + "time": "0:02:01.596363", + "diversity": { + "train": { + "mean_centered_corr": -0.019050154834985733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046319905668497086 + }, + "val": { + "mean_centered_corr": -0.018872274085879326, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06369400024414062 + }, + "test": { + "mean_centered_corr": -0.016154184937477112, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0944347083568573 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015532" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0.toml new file mode 100644 index 000000000..c12e9e918 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..96fc04b8d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2212921485786695, + "mae": 0.11681316047906876, + "r2": 0.6871429681777954, + "score": -0.2212921485786695 + }, + "val": { + "rmse": 0.23532450983472006, + "mae": 0.12748965620994568, + "r2": 0.6300989389419556, + "score": -0.23532450983472006 + }, + "test": { + "rmse": 0.2376596833156001, + "mae": 0.15176545083522797, + "r2": 0.6140326261520386, + "score": -0.2376596833156001 + } + }, + "time": "0:02:01.596363", + "diversity": { + "train": { + "mean_centered_corr": -0.019050154834985733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046319905668497086 + }, + "val": { + "mean_centered_corr": -0.018872274085879326, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06369400024414062 + }, + "test": { + "mean_centered_corr": -0.016154184937477112, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0944347083568573 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..96d88dd1b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,663 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22446870130378824, + "mae": 0.1184302344918251, + "r2": 0.6780966520309448, + "score": -0.22446870130378824 + }, + "val": { + "rmse": 0.23408501487488748, + "mae": 0.126699760556221, + "r2": 0.6339853405952454, + "score": -0.23408501487488748 + }, + "test": { + "rmse": 0.23576018763280351, + "mae": 0.14970247447490692, + "r2": 0.6201776266098022, + "score": -0.23576018763280351 + } + }, + "time": "0:02:01.596363", + "diversity": { + "train": { + "mean_centered_corr": -0.019050154834985733, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046319905668497086 + }, + "val": { + "mean_centered_corr": -0.018872274085879326, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06369400024414062 + }, + "test": { + "mean_centered_corr": -0.016154184937477112, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0944347083568573 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 8, + 21, + 29, + 6, + 22 + ], + "head_selection_time": "0:00:00.102858" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6db4b89df --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22990253367135435, + "mae": 0.12059187889099121, + "r2": 0.6623230576515198, + "score": -0.22990253367135435 + }, + "val": { + "rmse": 0.23844761590759667, + "mae": 0.1282438039779663, + "r2": 0.6202155351638794, + "score": -0.23844761590759667 + }, + "test": { + "rmse": 0.2416540932193429, + "mae": 0.14754453301429749, + "r2": 0.6009494066238403, + "score": -0.2416540932193429 + } + }, + "time": "0:02:02.101286", + "diversity": { + "train": { + "mean_centered_corr": -0.018734341487288475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044835660606622696 + }, + "val": { + "mean_centered_corr": -0.018935222178697586, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06088249757885933 + }, + "test": { + "mean_centered_corr": -0.01582489162683487, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08836037665605545 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.015062" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0.toml new file mode 100644 index 000000000..7de9c7b34 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..450021870 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2216062550188277, + "mae": 0.1181904599070549, + "r2": 0.6862542033195496, + "score": -0.2216062550188277 + }, + "val": { + "rmse": 0.23472067454979642, + "mae": 0.12802676856517792, + "r2": 0.6319948434829712, + "score": -0.23472067454979642 + }, + "test": { + "rmse": 0.23644700488851306, + "mae": 0.15162619948387146, + "r2": 0.6179614067077637, + "score": -0.23644700488851306 + } + }, + "time": "0:02:02.101286", + "diversity": { + "train": { + "mean_centered_corr": -0.018734341487288475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044835660606622696 + }, + "val": { + "mean_centered_corr": -0.018935222178697586, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06088249757885933 + }, + "test": { + "mean_centered_corr": -0.01582489162683487, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08836037665605545 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d21ed4c35 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,664 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2225587447309463, + "mae": 0.11954773962497711, + "r2": 0.6835513710975647, + "score": -0.2225587447309463 + }, + "val": { + "rmse": 0.23314779382394482, + "mae": 0.12646809220314026, + "r2": 0.6369103193283081, + "score": -0.23314779382394482 + }, + "test": { + "rmse": 0.23448866631035667, + "mae": 0.14718787372112274, + "r2": 0.6242635846138, + "score": -0.23448866631035667 + } + }, + "time": "0:02:02.101286", + "diversity": { + "train": { + "mean_centered_corr": -0.018734341487288475, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044835660606622696 + }, + "val": { + "mean_centered_corr": -0.018935222178697586, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06088249757885933 + }, + "test": { + "mean_centered_corr": -0.01582489162683487, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08836037665605545 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8, + 25, + 19, + 4, + 21, + 22, + 5 + ], + "head_selection_time": "0:00:00.116994" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e9aadf54d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23604626832625122, + "mae": 0.13103090226650238, + "r2": 0.6440342664718628, + "score": -0.23604626832625122 + }, + "val": { + "rmse": 0.2383569065742623, + "mae": 0.13819104433059692, + "r2": 0.6205043792724609, + "score": -0.2383569065742623 + }, + "test": { + "rmse": 0.244620972160157, + "mae": 0.16388729214668274, + "r2": 0.5910906791687012, + "score": -0.244620972160157 + } + }, + "time": "0:01:52.165374", + "diversity": { + "train": { + "mean_centered_corr": -0.018348872661590576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04553309082984924 + }, + "val": { + "mean_centered_corr": -0.019239172339439392, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05902950465679169 + }, + "test": { + "mean_centered_corr": -0.017977142706513405, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08112667500972748 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:00.015406" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0.toml new file mode 100644 index 000000000..0d76249b7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..bbac15ab2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23063516266200654, + "mae": 0.12075714766979218, + "r2": 0.6601674556732178, + "score": -0.23063516266200654 + }, + "val": { + "rmse": 0.23505473908329258, + "mae": 0.12712784111499786, + "r2": 0.6309465169906616, + "score": -0.23505473908329258 + }, + "test": { + "rmse": 0.23281250512039894, + "mae": 0.14543399214744568, + "r2": 0.6296160221099854, + "score": -0.23281250512039894 + } + }, + "time": "0:01:52.165374", + "diversity": { + "train": { + "mean_centered_corr": -0.018348872661590576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04553309082984924 + }, + "val": { + "mean_centered_corr": -0.019239172339439392, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05902950465679169 + }, + "test": { + "mean_centered_corr": -0.017977142706513405, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08112667500972748 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..700225951 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,666 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23284740122388672, + "mae": 0.1222502812743187, + "r2": 0.6536169052124023, + "score": -0.23284740122388672 + }, + "val": { + "rmse": 0.2334130414280393, + "mae": 0.12650004029273987, + "r2": 0.636083722114563, + "score": -0.2334130414280393 + }, + "test": { + "rmse": 0.23649611689361716, + "mae": 0.15280967950820923, + "r2": 0.6178027391433716, + "score": -0.23649611689361716 + } + }, + "time": "0:01:52.165374", + "diversity": { + "train": { + "mean_centered_corr": -0.018348872661590576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04553309082984924 + }, + "val": { + "mean_centered_corr": -0.019239172339439392, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05902950465679169 + }, + "test": { + "mean_centered_corr": -0.017977142706513405, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08112667500972748 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 1, + 10, + 25, + 21, + 28, + 19, + 0, + 1 + ], + "head_selection_time": "0:00:00.146443" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-best-head-evaluation/0/report.json new file mode 100644 index 000000000..fcf2bb9e1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-best-head-evaluation/0/report.json @@ -0,0 +1,658 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23952544677482882, + "mae": 0.12448744475841522, + "r2": 0.6334635019302368, + "score": -0.23952544677482882 + }, + "val": { + "rmse": 0.23747199640064248, + "mae": 0.13178245723247528, + "r2": 0.6233169436454773, + "score": -0.23747199640064248 + }, + "test": { + "rmse": 0.24725484176674034, + "mae": 0.16568316519260406, + "r2": 0.582237720489502, + "score": -0.24725484176674034 + } + }, + "time": "0:01:53.326611", + "diversity": { + "train": { + "mean_centered_corr": -0.018683521077036858, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04501955956220627 + }, + "val": { + "mean_centered_corr": -0.019395897164940834, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05899025872349739 + }, + "test": { + "mean_centered_corr": -0.017709530889987946, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0825389176607132 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015209" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0.toml new file mode 100644 index 000000000..e17dd27d1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0.toml @@ -0,0 +1,52 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.02 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0/report.json new file mode 100644 index 000000000..7982459ce --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-evaluation/0/report.json @@ -0,0 +1,654 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23193801072582776, + "mae": 0.1212281584739685, + "r2": 0.6563172340393066, + "score": -0.23193801072582776 + }, + "val": { + "rmse": 0.23511142271763394, + "mae": 0.12687157094478607, + "r2": 0.6307685375213623, + "score": -0.23511142271763394 + }, + "test": { + "rmse": 0.23473994135203347, + "mae": 0.14816908538341522, + "r2": 0.6234579086303711, + "score": -0.23473994135203347 + } + }, + "time": "0:01:53.326611", + "diversity": { + "train": { + "mean_centered_corr": -0.018683521077036858, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04501955956220627 + }, + "val": { + "mean_centered_corr": -0.019395897164940834, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05899025872349739 + }, + "test": { + "mean_centered_corr": -0.017709530889987946, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0825389176607132 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..192aaf7ed --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_cf_r4_n1em05_l0p02-greedy-heads-evaluation/0/report.json @@ -0,0 +1,665 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.02, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23442632590013812, + "mae": 0.1224464401602745, + "r2": 0.6489033699035645, + "score": -0.23442632590013812 + }, + "val": { + "rmse": 0.23366474775887086, + "mae": 0.12645675241947174, + "r2": 0.6352983713150024, + "score": -0.23366474775887086 + }, + "test": { + "rmse": 0.23646780882908092, + "mae": 0.15090638399124146, + "r2": 0.617894172668457, + "score": -0.23646780882908092 + } + }, + "time": "0:01:53.326611", + "diversity": { + "train": { + "mean_centered_corr": -0.018683521077036858, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04501955956220627 + }, + "val": { + "mean_centered_corr": -0.019395897164940834, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05899025872349739 + }, + "test": { + "mean_centered_corr": -0.017709530889987946, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0825389176607132 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 29, + 25, + 10, + 1, + 0, + 19, + 28 + ], + "head_selection_time": "0:00:00.130044" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..69b541d02 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23782740036230163, + "mae": 0.13576887547969818, + "r2": 0.6386420130729675, + "score": -0.23782740036230163 + }, + "val": { + "rmse": 0.23857661998288315, + "mae": 0.13661375641822815, + "r2": 0.6198044419288635, + "score": -0.23857661998288315 + }, + "test": { + "rmse": 0.2544202590204169, + "mae": 0.17306958138942719, + "r2": 0.557673454284668, + "score": -0.2544202590204169 + } + }, + "time": "0:01:21.016540", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015229" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..e8b03a128 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..4e423b1d6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23108709757774937, + "mae": 0.1223861500620842, + "r2": 0.6588343381881714, + "score": -0.23108709757774937 + }, + "val": { + "rmse": 0.23484986994593354, + "mae": 0.12684030830860138, + "r2": 0.6315895318984985, + "score": -0.23484986994593354 + }, + "test": { + "rmse": 0.23462197483665095, + "mae": 0.1482354700565338, + "r2": 0.6238362193107605, + "score": -0.23462197483665095 + } + }, + "time": "0:01:21.016540", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..dae752a86 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,217 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2319917866834578, + "mae": 0.12356825172901154, + "r2": 0.6561578512191772, + "score": -0.2319917866834578 + }, + "val": { + "rmse": 0.23320292020753594, + "mae": 0.12550441920757294, + "r2": 0.636738657951355, + "score": -0.23320292020753594 + }, + "test": { + "rmse": 0.23031042609193297, + "mae": 0.14081889390945435, + "r2": 0.6375343799591064, + "score": -0.23031042609193297 + } + }, + "time": "0:01:21.016540", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 4, + 28, + 1, + 19, + 24, + 16, + 4, + 1, + 20, + 21 + ], + "head_selection_time": "0:00:00.174232" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3d8fb6589 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2320922951078084, + "mae": 0.12688623368740082, + "r2": 0.6558598279953003, + "score": -0.2320922951078084 + }, + "val": { + "rmse": 0.2390755017179478, + "mae": 0.13819952309131622, + "r2": 0.6182127594947815, + "score": -0.2390755017179478 + }, + "test": { + "rmse": 0.25168892959044736, + "mae": 0.1720852106809616, + "r2": 0.5671197175979614, + "score": -0.25168892959044736 + } + }, + "time": "0:01:27.443113", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.015348" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..e87934418 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..45713e42c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2222445177940426, + "mae": 0.12026011943817139, + "r2": 0.6844443082809448, + "score": -0.2222445177940426 + }, + "val": { + "rmse": 0.23510992538054096, + "mae": 0.12966062128543854, + "r2": 0.6307731866836548, + "score": -0.23510992538054096 + }, + "test": { + "rmse": 0.24175341990615093, + "mae": 0.1596364676952362, + "r2": 0.6006213426589966, + "score": -0.24175341990615093 + } + }, + "time": "0:01:27.443113", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..aaca2bced --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22296470056985443, + "mae": 0.12100711464881897, + "r2": 0.682395875453949, + "score": -0.22296470056985443 + }, + "val": { + "rmse": 0.2333487134502395, + "mae": 0.12775763869285583, + "r2": 0.6362842321395874, + "score": -0.2333487134502395 + }, + "test": { + "rmse": 0.2416932153437863, + "mae": 0.16084370017051697, + "r2": 0.6008202433586121, + "score": -0.2416932153437863 + } + }, + "time": "0:01:27.443113", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 3, + 4, + 8, + 25, + 20, + 21 + ], + "head_selection_time": "0:00:00.116276" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..01cc37386 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2255253318194067, + "mae": 0.1253618746995926, + "r2": 0.6750589609146118, + "score": -0.2255253318194067 + }, + "val": { + "rmse": 0.23949031043262045, + "mae": 0.13341857492923737, + "r2": 0.6168867349624634, + "score": -0.23949031043262045 + }, + "test": { + "rmse": 0.24155888192007524, + "mae": 0.14752615988254547, + "r2": 0.6012638807296753, + "score": -0.24155888192007524 + } + }, + "time": "0:01:30.844410", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015134" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..b3d60a0de --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..c14e16500 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2197185345348766, + "mae": 0.11797432601451874, + "r2": 0.6915766000747681, + "score": -0.2197185345348766 + }, + "val": { + "rmse": 0.23467774706155475, + "mae": 0.1267256885766983, + "r2": 0.632129430770874, + "score": -0.23467774706155475 + }, + "test": { + "rmse": 0.23716614057842658, + "mae": 0.1508055180311203, + "r2": 0.6156339645385742, + "score": -0.23716614057842658 + } + }, + "time": "0:01:30.844410", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f6a84f3d5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22057000931073897, + "mae": 0.11852128803730011, + "r2": 0.6891815662384033, + "score": -0.22057000931073897 + }, + "val": { + "rmse": 0.23316146280351166, + "mae": 0.12596334517002106, + "r2": 0.6368677616119385, + "score": -0.23316146280351166 + }, + "test": { + "rmse": 0.23452524296935331, + "mae": 0.14668044447898865, + "r2": 0.6241463422775269, + "score": -0.23452524296935331 + } + }, + "time": "0:01:30.844410", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 4, + 16, + 12, + 0, + 25, + 31, + 3, + 22, + 29, + 12, + 27, + 20 + ], + "head_selection_time": "0:00:00.204011" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bed297dff --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2261742520907439, + "mae": 0.12224072217941284, + "r2": 0.6731863021850586, + "score": -0.2261742520907439 + }, + "val": { + "rmse": 0.2377136850270048, + "mae": 0.13327811658382416, + "r2": 0.622549831867218, + "score": -0.2377136850270048 + }, + "test": { + "rmse": 0.262345673404661, + "mae": 0.17990553379058838, + "r2": 0.5296865105628967, + "score": -0.262345673404661 + } + }, + "time": "0:01:26.676984", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015248" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..8d6397619 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..0ce8dd477 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22265827445732783, + "mae": 0.11788544058799744, + "r2": 0.6832683086395264, + "score": -0.22265827445732783 + }, + "val": { + "rmse": 0.23474623366768377, + "mae": 0.1270996779203415, + "r2": 0.6319146752357483, + "score": -0.23474623366768377 + }, + "test": { + "rmse": 0.23936563514587833, + "mae": 0.15535929799079895, + "r2": 0.6084716320037842, + "score": -0.23936563514587833 + } + }, + "time": "0:01:26.676984", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..bc918699d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,214 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22450348389711117, + "mae": 0.11922673135995865, + "r2": 0.6779968738555908, + "score": -0.22450348389711117 + }, + "val": { + "rmse": 0.23295802298295287, + "mae": 0.1258699893951416, + "r2": 0.6375011801719666, + "score": -0.23295802298295287 + }, + "test": { + "rmse": 0.23903366020873654, + "mae": 0.15511320531368256, + "r2": 0.6095569133758545, + "score": -0.23903366020873654 + } + }, + "time": "0:01:26.676984", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 22, + 12, + 16, + 21, + 19, + 25, + 29 + ], + "head_selection_time": "0:00:00.129748" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c6c0cd90a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23782740036230163, + "mae": 0.13576887547969818, + "r2": 0.6386420130729675, + "score": -0.23782740036230163 + }, + "val": { + "rmse": 0.23857661998288315, + "mae": 0.13661375641822815, + "r2": 0.6198044419288635, + "score": -0.23857661998288315 + }, + "test": { + "rmse": 0.2544202590204169, + "mae": 0.17306958138942719, + "r2": 0.557673454284668, + "score": -0.2544202590204169 + } + }, + "time": "0:01:20.336729", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015352" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..5fdf43534 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..e5dff772e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23108709757774937, + "mae": 0.1223861500620842, + "r2": 0.6588343381881714, + "score": -0.23108709757774937 + }, + "val": { + "rmse": 0.23484986994593354, + "mae": 0.12684030830860138, + "r2": 0.6315895318984985, + "score": -0.23484986994593354 + }, + "test": { + "rmse": 0.23462197483665095, + "mae": 0.1482354700565338, + "r2": 0.6238362193107605, + "score": -0.23462197483665095 + } + }, + "time": "0:01:20.336729", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d5837e7f3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,217 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2319917866834578, + "mae": 0.12356825172901154, + "r2": 0.6561578512191772, + "score": -0.2319917866834578 + }, + "val": { + "rmse": 0.23320292020753594, + "mae": 0.12550441920757294, + "r2": 0.636738657951355, + "score": -0.23320292020753594 + }, + "test": { + "rmse": 0.23031042609193297, + "mae": 0.14081889390945435, + "r2": 0.6375343799591064, + "score": -0.23031042609193297 + } + }, + "time": "0:01:20.336729", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 4, + 28, + 1, + 19, + 24, + 16, + 4, + 1, + 20, + 21 + ], + "head_selection_time": "0:00:00.174065" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0efc6a4da --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2320922951078084, + "mae": 0.12688623368740082, + "r2": 0.6558598279953003, + "score": -0.2320922951078084 + }, + "val": { + "rmse": 0.2390755017179478, + "mae": 0.13819952309131622, + "r2": 0.6182127594947815, + "score": -0.2390755017179478 + }, + "test": { + "rmse": 0.25168892959044736, + "mae": 0.1720852106809616, + "r2": 0.5671197175979614, + "score": -0.25168892959044736 + } + }, + "time": "0:01:27.032752", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.015405" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..a2f9fafcb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..2ab000289 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2222445177940426, + "mae": 0.12026011943817139, + "r2": 0.6844443082809448, + "score": -0.2222445177940426 + }, + "val": { + "rmse": 0.23510992538054096, + "mae": 0.12966062128543854, + "r2": 0.6307731866836548, + "score": -0.23510992538054096 + }, + "test": { + "rmse": 0.24175341990615093, + "mae": 0.1596364676952362, + "r2": 0.6006213426589966, + "score": -0.24175341990615093 + } + }, + "time": "0:01:27.032752", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..93707571d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22296470056985443, + "mae": 0.12100711464881897, + "r2": 0.682395875453949, + "score": -0.22296470056985443 + }, + "val": { + "rmse": 0.2333487134502395, + "mae": 0.12775763869285583, + "r2": 0.6362842321395874, + "score": -0.2333487134502395 + }, + "test": { + "rmse": 0.2416932153437863, + "mae": 0.16084370017051697, + "r2": 0.6008202433586121, + "score": -0.2416932153437863 + } + }, + "time": "0:01:27.032752", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 3, + 4, + 8, + 25, + 20, + 21 + ], + "head_selection_time": "0:00:00.116690" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ba45263d6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2255253318194067, + "mae": 0.1253618746995926, + "r2": 0.6750589609146118, + "score": -0.2255253318194067 + }, + "val": { + "rmse": 0.23949031043262045, + "mae": 0.13341857492923737, + "r2": 0.6168867349624634, + "score": -0.23949031043262045 + }, + "test": { + "rmse": 0.24155888192007524, + "mae": 0.14752615988254547, + "r2": 0.6012638807296753, + "score": -0.24155888192007524 + } + }, + "time": "0:01:29.541702", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015004" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..3ad7b4def --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..51b5864bf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2197185345348766, + "mae": 0.11797432601451874, + "r2": 0.6915766000747681, + "score": -0.2197185345348766 + }, + "val": { + "rmse": 0.23467774706155475, + "mae": 0.1267256885766983, + "r2": 0.632129430770874, + "score": -0.23467774706155475 + }, + "test": { + "rmse": 0.23716614057842658, + "mae": 0.1508055180311203, + "r2": 0.6156339645385742, + "score": -0.23716614057842658 + } + }, + "time": "0:01:29.541702", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4a0a5ed75 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22057000931073897, + "mae": 0.11852128803730011, + "r2": 0.6891815662384033, + "score": -0.22057000931073897 + }, + "val": { + "rmse": 0.23316146280351166, + "mae": 0.12596334517002106, + "r2": 0.6368677616119385, + "score": -0.23316146280351166 + }, + "test": { + "rmse": 0.23452524296935331, + "mae": 0.14668044447898865, + "r2": 0.6241463422775269, + "score": -0.23452524296935331 + } + }, + "time": "0:01:29.541702", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 4, + 16, + 12, + 0, + 25, + 31, + 3, + 22, + 29, + 12, + 27, + 20 + ], + "head_selection_time": "0:00:00.203091" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bbb2b37fa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2261742520907439, + "mae": 0.12224072217941284, + "r2": 0.6731863021850586, + "score": -0.2261742520907439 + }, + "val": { + "rmse": 0.2377136850270048, + "mae": 0.13327811658382416, + "r2": 0.622549831867218, + "score": -0.2377136850270048 + }, + "test": { + "rmse": 0.262345673404661, + "mae": 0.17990553379058838, + "r2": 0.5296865105628967, + "score": -0.262345673404661 + } + }, + "time": "0:01:27.105568", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015126" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..d9ba67e38 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..fca4bfe7b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22265827445732783, + "mae": 0.11788544058799744, + "r2": 0.6832683086395264, + "score": -0.22265827445732783 + }, + "val": { + "rmse": 0.23474623366768377, + "mae": 0.1270996779203415, + "r2": 0.6319146752357483, + "score": -0.23474623366768377 + }, + "test": { + "rmse": 0.23936563514587833, + "mae": 0.15535929799079895, + "r2": 0.6084716320037842, + "score": -0.23936563514587833 + } + }, + "time": "0:01:27.105568", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..81108d9f0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,214 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p0001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22450348389711117, + "mae": 0.11922673135995865, + "r2": 0.6779968738555908, + "score": -0.22450348389711117 + }, + "val": { + "rmse": 0.23295802298295287, + "mae": 0.1258699893951416, + "r2": 0.6375011801719666, + "score": -0.23295802298295287 + }, + "test": { + "rmse": 0.23903366020873654, + "mae": 0.15511320531368256, + "r2": 0.6095569133758545, + "score": -0.23903366020873654 + } + }, + "time": "0:01:27.105568", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 22, + 12, + 16, + 21, + 19, + 25, + 29 + ], + "head_selection_time": "0:00:00.129037" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b8acbff5c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23782740036230163, + "mae": 0.13576887547969818, + "r2": 0.6386420130729675, + "score": -0.23782740036230163 + }, + "val": { + "rmse": 0.23857661998288315, + "mae": 0.13661375641822815, + "r2": 0.6198044419288635, + "score": -0.23857661998288315 + }, + "test": { + "rmse": 0.2544202590204169, + "mae": 0.17306958138942719, + "r2": 0.557673454284668, + "score": -0.2544202590204169 + } + }, + "time": "0:01:20.386711", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015002" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..2f8be345d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..8b8f2f72d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23108709757774937, + "mae": 0.1223861500620842, + "r2": 0.6588343381881714, + "score": -0.23108709757774937 + }, + "val": { + "rmse": 0.23484986994593354, + "mae": 0.12684030830860138, + "r2": 0.6315895318984985, + "score": -0.23484986994593354 + }, + "test": { + "rmse": 0.23462197483665095, + "mae": 0.1482354700565338, + "r2": 0.6238362193107605, + "score": -0.23462197483665095 + } + }, + "time": "0:01:20.386711", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..afa6f99ff --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,217 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2319917866834578, + "mae": 0.12356825172901154, + "r2": 0.6561578512191772, + "score": -0.2319917866834578 + }, + "val": { + "rmse": 0.23320292020753594, + "mae": 0.12550441920757294, + "r2": 0.636738657951355, + "score": -0.23320292020753594 + }, + "test": { + "rmse": 0.23031042609193297, + "mae": 0.14081889390945435, + "r2": 0.6375343799591064, + "score": -0.23031042609193297 + } + }, + "time": "0:01:20.386711", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 4, + 28, + 1, + 19, + 24, + 16, + 4, + 1, + 20, + 21 + ], + "head_selection_time": "0:00:00.171966" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1e9aadbf5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2320922951078084, + "mae": 0.12688623368740082, + "r2": 0.6558598279953003, + "score": -0.2320922951078084 + }, + "val": { + "rmse": 0.2390755017179478, + "mae": 0.13819952309131622, + "r2": 0.6182127594947815, + "score": -0.2390755017179478 + }, + "test": { + "rmse": 0.25168892959044736, + "mae": 0.1720852106809616, + "r2": 0.5671197175979614, + "score": -0.25168892959044736 + } + }, + "time": "0:01:27.604837", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.015528" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..237bd5384 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..8110cb0cf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2222445177940426, + "mae": 0.12026011943817139, + "r2": 0.6844443082809448, + "score": -0.2222445177940426 + }, + "val": { + "rmse": 0.23510992538054096, + "mae": 0.12966062128543854, + "r2": 0.6307731866836548, + "score": -0.23510992538054096 + }, + "test": { + "rmse": 0.24175341990615093, + "mae": 0.1596364676952362, + "r2": 0.6006213426589966, + "score": -0.24175341990615093 + } + }, + "time": "0:01:27.604837", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f570919e0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22296470056985443, + "mae": 0.12100711464881897, + "r2": 0.682395875453949, + "score": -0.22296470056985443 + }, + "val": { + "rmse": 0.2333487134502395, + "mae": 0.12775763869285583, + "r2": 0.6362842321395874, + "score": -0.2333487134502395 + }, + "test": { + "rmse": 0.2416932153437863, + "mae": 0.16084370017051697, + "r2": 0.6008202433586121, + "score": -0.2416932153437863 + } + }, + "time": "0:01:27.604837", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 3, + 4, + 8, + 25, + 20, + 21 + ], + "head_selection_time": "0:00:00.115215" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8844d39fb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2255253318194067, + "mae": 0.1253618746995926, + "r2": 0.6750589609146118, + "score": -0.2255253318194067 + }, + "val": { + "rmse": 0.23949031043262045, + "mae": 0.13341857492923737, + "r2": 0.6168867349624634, + "score": -0.23949031043262045 + }, + "test": { + "rmse": 0.24155888192007524, + "mae": 0.14752615988254547, + "r2": 0.6012638807296753, + "score": -0.24155888192007524 + } + }, + "time": "0:01:30.152123", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015102" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..d65a74bc0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..12a205021 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2197185345348766, + "mae": 0.11797432601451874, + "r2": 0.6915766000747681, + "score": -0.2197185345348766 + }, + "val": { + "rmse": 0.23467774706155475, + "mae": 0.1267256885766983, + "r2": 0.632129430770874, + "score": -0.23467774706155475 + }, + "test": { + "rmse": 0.23716614057842658, + "mae": 0.1508055180311203, + "r2": 0.6156339645385742, + "score": -0.23716614057842658 + } + }, + "time": "0:01:30.152123", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9d2512c60 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22057000931073897, + "mae": 0.11852128803730011, + "r2": 0.6891815662384033, + "score": -0.22057000931073897 + }, + "val": { + "rmse": 0.23316146280351166, + "mae": 0.12596334517002106, + "r2": 0.6368677616119385, + "score": -0.23316146280351166 + }, + "test": { + "rmse": 0.23452524296935331, + "mae": 0.14668044447898865, + "r2": 0.6241463422775269, + "score": -0.23452524296935331 + } + }, + "time": "0:01:30.152123", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 4, + 16, + 12, + 0, + 25, + 31, + 3, + 22, + 29, + 12, + 27, + 20 + ], + "head_selection_time": "0:00:00.202186" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7e8a2da9b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2261742520907439, + "mae": 0.12224072217941284, + "r2": 0.6731863021850586, + "score": -0.2261742520907439 + }, + "val": { + "rmse": 0.2377136850270048, + "mae": 0.13327811658382416, + "r2": 0.622549831867218, + "score": -0.2377136850270048 + }, + "test": { + "rmse": 0.262345673404661, + "mae": 0.17990553379058838, + "r2": 0.5296865105628967, + "score": -0.262345673404661 + } + }, + "time": "0:01:26.505665", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015205" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..f5afb1e37 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..825c4e2d0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22265827445732783, + "mae": 0.11788544058799744, + "r2": 0.6832683086395264, + "score": -0.22265827445732783 + }, + "val": { + "rmse": 0.23474623366768377, + "mae": 0.1270996779203415, + "r2": 0.6319146752357483, + "score": -0.23474623366768377 + }, + "test": { + "rmse": 0.23936563514587833, + "mae": 0.15535929799079895, + "r2": 0.6084716320037842, + "score": -0.23936563514587833 + } + }, + "time": "0:01:26.505665", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..325697f2f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,214 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n0p001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22450348389711117, + "mae": 0.11922673135995865, + "r2": 0.6779968738555908, + "score": -0.22450348389711117 + }, + "val": { + "rmse": 0.23295802298295287, + "mae": 0.1258699893951416, + "r2": 0.6375011801719666, + "score": -0.23295802298295287 + }, + "test": { + "rmse": 0.23903366020873654, + "mae": 0.15511320531368256, + "r2": 0.6095569133758545, + "score": -0.23903366020873654 + } + }, + "time": "0:01:26.505665", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 22, + 12, + 16, + 21, + 19, + 25, + 29 + ], + "head_selection_time": "0:00:00.128994" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..d54a7312a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23782740036230163, + "mae": 0.13576887547969818, + "r2": 0.6386420130729675, + "score": -0.23782740036230163 + }, + "val": { + "rmse": 0.23857661998288315, + "mae": 0.13661375641822815, + "r2": 0.6198044419288635, + "score": -0.23857661998288315 + }, + "test": { + "rmse": 0.2544202590204169, + "mae": 0.17306958138942719, + "r2": 0.557673454284668, + "score": -0.2544202590204169 + } + }, + "time": "0:01:21.063765", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015192" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..8bf510609 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..4acc7e1b1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23108709757774937, + "mae": 0.1223861500620842, + "r2": 0.6588343381881714, + "score": -0.23108709757774937 + }, + "val": { + "rmse": 0.23484986994593354, + "mae": 0.12684030830860138, + "r2": 0.6315895318984985, + "score": -0.23484986994593354 + }, + "test": { + "rmse": 0.23462197483665095, + "mae": 0.1482354700565338, + "r2": 0.6238362193107605, + "score": -0.23462197483665095 + } + }, + "time": "0:01:21.063765", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2956d8bbc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,217 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2319917866834578, + "mae": 0.12356825172901154, + "r2": 0.6561578512191772, + "score": -0.2319917866834578 + }, + "val": { + "rmse": 0.23320292020753594, + "mae": 0.12550441920757294, + "r2": 0.636738657951355, + "score": -0.23320292020753594 + }, + "test": { + "rmse": 0.23031042609193297, + "mae": 0.14081889390945435, + "r2": 0.6375343799591064, + "score": -0.23031042609193297 + } + }, + "time": "0:01:21.063765", + "diversity": { + "train": { + "mean_centered_corr": -0.01925019733607769, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815442115068436 + }, + "val": { + "mean_centered_corr": -0.018709633499383926, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06300598382949829 + }, + "test": { + "mean_centered_corr": -0.018054556101560593, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491437882184982 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 4, + 28, + 1, + 19, + 24, + 16, + 4, + 1, + 20, + 21 + ], + "head_selection_time": "0:00:00.173704" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a4081fa3f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2320922951078084, + "mae": 0.12688623368740082, + "r2": 0.6558598279953003, + "score": -0.2320922951078084 + }, + "val": { + "rmse": 0.2390755017179478, + "mae": 0.13819952309131622, + "r2": 0.6182127594947815, + "score": -0.2390755017179478 + }, + "test": { + "rmse": 0.25168892959044736, + "mae": 0.1720852106809616, + "r2": 0.5671197175979614, + "score": -0.25168892959044736 + } + }, + "time": "0:01:27.149432", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.015214" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..e8b20d26c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..baf1310ef --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2222445177940426, + "mae": 0.12026011943817139, + "r2": 0.6844443082809448, + "score": -0.2222445177940426 + }, + "val": { + "rmse": 0.23510992538054096, + "mae": 0.12966062128543854, + "r2": 0.6307731866836548, + "score": -0.23510992538054096 + }, + "test": { + "rmse": 0.24175341990615093, + "mae": 0.1596364676952362, + "r2": 0.6006213426589966, + "score": -0.24175341990615093 + } + }, + "time": "0:01:27.149432", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4a4d3ce2f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22296470056985443, + "mae": 0.12100711464881897, + "r2": 0.682395875453949, + "score": -0.22296470056985443 + }, + "val": { + "rmse": 0.2333487134502395, + "mae": 0.12775763869285583, + "r2": 0.6362842321395874, + "score": -0.2333487134502395 + }, + "test": { + "rmse": 0.2416932153437863, + "mae": 0.16084370017051697, + "r2": 0.6008202433586121, + "score": -0.2416932153437863 + } + }, + "time": "0:01:27.149432", + "diversity": { + "train": { + "mean_centered_corr": -0.018731974065303802, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049796536564826965 + }, + "val": { + "mean_centered_corr": -0.019370803609490395, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06845120340585709 + }, + "test": { + "mean_centered_corr": -0.016793658956885338, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0947653204202652 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 3, + 4, + 8, + 25, + 20, + 21 + ], + "head_selection_time": "0:00:00.115778" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..67f1062dc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2255253318194067, + "mae": 0.1253618746995926, + "r2": 0.6750589609146118, + "score": -0.2255253318194067 + }, + "val": { + "rmse": 0.23949031043262045, + "mae": 0.13341857492923737, + "r2": 0.6168867349624634, + "score": -0.23949031043262045 + }, + "test": { + "rmse": 0.24155888192007524, + "mae": 0.14752615988254547, + "r2": 0.6012638807296753, + "score": -0.24155888192007524 + } + }, + "time": "0:01:30.041102", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015164" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..bfd17e870 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..df0136fbb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2197185345348766, + "mae": 0.11797432601451874, + "r2": 0.6915766000747681, + "score": -0.2197185345348766 + }, + "val": { + "rmse": 0.23467774706155475, + "mae": 0.1267256885766983, + "r2": 0.632129430770874, + "score": -0.23467774706155475 + }, + "test": { + "rmse": 0.23716614057842658, + "mae": 0.1508055180311203, + "r2": 0.6156339645385742, + "score": -0.23716614057842658 + } + }, + "time": "0:01:30.041102", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7d9441f30 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22057000931073897, + "mae": 0.11852128803730011, + "r2": 0.6891815662384033, + "score": -0.22057000931073897 + }, + "val": { + "rmse": 0.23316146280351166, + "mae": 0.12596334517002106, + "r2": 0.6368677616119385, + "score": -0.23316146280351166 + }, + "test": { + "rmse": 0.23452524296935331, + "mae": 0.14668044447898865, + "r2": 0.6241463422775269, + "score": -0.23452524296935331 + } + }, + "time": "0:01:30.041102", + "diversity": { + "train": { + "mean_centered_corr": -0.017563171684741974, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048281844705343246 + }, + "val": { + "mean_centered_corr": -0.01754843257367611, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07056254893541336 + }, + "test": { + "mean_centered_corr": -0.016294721513986588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09666775166988373 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 4, + 16, + 12, + 0, + 25, + 31, + 3, + 22, + 29, + 12, + 27, + 20 + ], + "head_selection_time": "0:00:00.205114" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9a13723e7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,207 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2261742520907439, + "mae": 0.12224072217941284, + "r2": 0.6731863021850586, + "score": -0.2261742520907439 + }, + "val": { + "rmse": 0.2377136850270048, + "mae": 0.13327811658382416, + "r2": 0.622549831867218, + "score": -0.2377136850270048 + }, + "test": { + "rmse": 0.262345673404661, + "mae": 0.17990553379058838, + "r2": 0.5296865105628967, + "score": -0.262345673404661 + } + }, + "time": "0:01:27.660572", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015449" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..7a8746abc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..f597ea970 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/0/report.json @@ -0,0 +1,203 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22265827445732783, + "mae": 0.11788544058799744, + "r2": 0.6832683086395264, + "score": -0.22265827445732783 + }, + "val": { + "rmse": 0.23474623366768377, + "mae": 0.1270996779203415, + "r2": 0.6319146752357483, + "score": -0.23474623366768377 + }, + "test": { + "rmse": 0.23936563514587833, + "mae": 0.15535929799079895, + "r2": 0.6084716320037842, + "score": -0.23936563514587833 + } + }, + "time": "0:01:27.660572", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8b959ea4c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,214 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r1_n1em05_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22450348389711117, + "mae": 0.11922673135995865, + "r2": 0.6779968738555908, + "score": -0.22450348389711117 + }, + "val": { + "rmse": 0.23295802298295287, + "mae": 0.1258699893951416, + "r2": 0.6375011801719666, + "score": -0.23295802298295287 + }, + "test": { + "rmse": 0.23903366020873654, + "mae": 0.15511320531368256, + "r2": 0.6095569133758545, + "score": -0.23903366020873654 + } + }, + "time": "0:01:27.660572", + "diversity": { + "train": { + "mean_centered_corr": -0.01859835349023342, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04920431599020958 + }, + "val": { + "mean_centered_corr": -0.019710257649421692, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06811030209064484 + }, + "test": { + "mean_centered_corr": -0.013359437696635723, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09920918941497803 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 22, + 12, + 16, + 21, + 19, + 25, + 29 + ], + "head_selection_time": "0:00:00.133344" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..92fd4c825 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 962, + "metrics": { + "train": { + "rmse": 0.22433421654217983, + "mae": 0.12722031772136688, + "r2": 0.6784822940826416, + "score": -0.22433421654217983 + }, + "val": { + "rmse": 0.24006717570021247, + "mae": 0.1350351870059967, + "r2": 0.6150389313697815, + "score": -0.24006717570021247 + }, + "test": { + "rmse": 0.24244266573296758, + "mae": 0.14548884332180023, + "r2": 0.5983408689498901, + "score": -0.24244266573296758 + } + }, + "time": "0:02:07.704029", + "diversity": { + "train": { + "mean_centered_corr": -0.016253771260380745, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049984753131866455 + }, + "val": { + "mean_centered_corr": -0.016636746004223824, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0729350671172142 + }, + "test": { + "mean_centered_corr": -0.01810598000884056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10638274252414703 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015142" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..f76490e2b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..c8df19d94 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 962, + "metrics": { + "train": { + "rmse": 0.21573275520444352, + "mae": 0.12005273252725601, + "r2": 0.7026649713516235, + "score": -0.21573275520444352 + }, + "val": { + "rmse": 0.23499185132041592, + "mae": 0.12818463146686554, + "r2": 0.6311439871788025, + "score": -0.23499185132041592 + }, + "test": { + "rmse": 0.2394094259267782, + "mae": 0.15464657545089722, + "r2": 0.6083284020423889, + "score": -0.2394094259267782 + } + }, + "time": "0:02:07.704029", + "diversity": { + "train": { + "mean_centered_corr": -0.016253771260380745, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049984753131866455 + }, + "val": { + "mean_centered_corr": -0.016636746004223824, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0729350671172142 + }, + "test": { + "mean_centered_corr": -0.01810598000884056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10638274252414703 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2fa26438c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 962, + "metrics": { + "train": { + "rmse": 0.21682843846304425, + "mae": 0.12020629644393921, + "r2": 0.6996370553970337, + "score": -0.21682843846304425 + }, + "val": { + "rmse": 0.23372927564914306, + "mae": 0.1274442821741104, + "r2": 0.6350969672203064, + "score": -0.23372927564914306 + }, + "test": { + "rmse": 0.2350238164160606, + "mae": 0.14731569588184357, + "r2": 0.6225466132164001, + "score": -0.2350238164160606 + } + }, + "time": "0:02:07.704029", + "diversity": { + "train": { + "mean_centered_corr": -0.016253771260380745, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049984753131866455 + }, + "val": { + "mean_centered_corr": -0.016636746004223824, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0729350671172142 + }, + "test": { + "mean_centered_corr": -0.01810598000884056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10638274252414703 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 16, + 20, + 21, + 4, + 12, + 27, + 28, + 2, + 22, + 31, + 8 + ], + "head_selection_time": "0:00:00.189880" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bdf7ec1d9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23104360784438088, + "mae": 0.12508618831634521, + "r2": 0.6589627265930176, + "score": -0.23104360784438088 + }, + "val": { + "rmse": 0.2383891939899224, + "mae": 0.13487277925014496, + "r2": 0.6204016208648682, + "score": -0.2383891939899224 + }, + "test": { + "rmse": 0.24401096715766643, + "mae": 0.15994176268577576, + "r2": 0.5931275486946106, + "score": -0.24401096715766643 + } + }, + "time": "0:01:59.937521", + "diversity": { + "train": { + "mean_centered_corr": -0.01880013383924961, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04820944741368294 + }, + "val": { + "mean_centered_corr": -0.01936255395412445, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06788419187068939 + }, + "test": { + "mean_centered_corr": -0.017522644251585007, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09551511704921722 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.015282" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..6312a247d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..30c1cc9a8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22253845679462023, + "mae": 0.11854175478219986, + "r2": 0.6836090683937073, + "score": -0.22253845679462023 + }, + "val": { + "rmse": 0.2346688971022881, + "mae": 0.12585745751857758, + "r2": 0.6321571469306946, + "score": -0.2346688971022881 + }, + "test": { + "rmse": 0.23375485551605374, + "mae": 0.14507132768630981, + "r2": 0.626611590385437, + "score": -0.23375485551605374 + } + }, + "time": "0:01:59.937521", + "diversity": { + "train": { + "mean_centered_corr": -0.01880013383924961, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04820944741368294 + }, + "val": { + "mean_centered_corr": -0.01936255395412445, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06788419187068939 + }, + "test": { + "mean_centered_corr": -0.017522644251585007, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09551511704921722 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1ec0b4622 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,221 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22311397819116993, + "mae": 0.12033727020025253, + "r2": 0.681970477104187, + "score": -0.22311397819116993 + }, + "val": { + "rmse": 0.23289370528688721, + "mae": 0.1270957887172699, + "r2": 0.6377012729644775, + "score": -0.23289370528688721 + }, + "test": { + "rmse": 0.2404508003490849, + "mae": 0.15944166481494904, + "r2": 0.6049136519432068, + "score": -0.2404508003490849 + } + }, + "time": "0:01:59.937521", + "diversity": { + "train": { + "mean_centered_corr": -0.01880013383924961, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04820944741368294 + }, + "val": { + "mean_centered_corr": -0.01936255395412445, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06788419187068939 + }, + "test": { + "mean_centered_corr": -0.017522644251585007, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09551511704921722 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 4, + 25, + 28, + 12, + 1, + 18, + 4, + 16 + ], + "head_selection_time": "0:00:00.144082" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ac1e0d186 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22728253913691077, + "mae": 0.12302348017692566, + "r2": 0.6699756383895874, + "score": -0.22728253913691077 + }, + "val": { + "rmse": 0.23897535038319878, + "mae": 0.13313278555870056, + "r2": 0.6185325384140015, + "score": -0.23897535038319878 + }, + "test": { + "rmse": 0.24383994851478993, + "mae": 0.14601385593414307, + "r2": 0.5936976671218872, + "score": -0.24383994851478993 + } + }, + "time": "0:01:59.114238", + "diversity": { + "train": { + "mean_centered_corr": -0.018884949386119843, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047628145664930344 + }, + "val": { + "mean_centered_corr": -0.017485102638602257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06956880539655685 + }, + "test": { + "mean_centered_corr": -0.01598832942545414, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09292414039373398 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015354" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..812605f91 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..46a42870c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2233263689720867, + "mae": 0.11833805590867996, + "r2": 0.6813646554946899, + "score": -0.2233263689720867 + }, + "val": { + "rmse": 0.23475735787864294, + "mae": 0.12769214808940887, + "r2": 0.6318798065185547, + "score": -0.23475735787864294 + }, + "test": { + "rmse": 0.23961932837948463, + "mae": 0.15490485727787018, + "r2": 0.607641339302063, + "score": -0.23961932837948463 + } + }, + "time": "0:01:59.114238", + "diversity": { + "train": { + "mean_centered_corr": -0.018884949386119843, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047628145664930344 + }, + "val": { + "mean_centered_corr": -0.017485102638602257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06956880539655685 + }, + "test": { + "mean_centered_corr": -0.01598832942545414, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09292414039373398 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ed5eb58b1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22469448467375502, + "mae": 0.12008367478847504, + "r2": 0.6774487495422363, + "score": -0.22469448467375502 + }, + "val": { + "rmse": 0.23343394819363691, + "mae": 0.12686417996883392, + "r2": 0.6360185146331787, + "score": -0.23343394819363691 + }, + "test": { + "rmse": 0.2365000863679966, + "mae": 0.15199944376945496, + "r2": 0.6177898645401001, + "score": -0.2365000863679966 + } + }, + "time": "0:01:59.114238", + "diversity": { + "train": { + "mean_centered_corr": -0.018884949386119843, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047628145664930344 + }, + "val": { + "mean_centered_corr": -0.017485102638602257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06956880539655685 + }, + "test": { + "mean_centered_corr": -0.01598832942545414, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09292414039373398 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 21, + 4, + 12, + 16, + 28, + 20, + 29, + 18, + 12, + 16, + 21 + ], + "head_selection_time": "0:00:00.192700" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7beab8ba3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2255459539475908, + "mae": 0.12257809191942215, + "r2": 0.6749995350837708, + "score": -0.2255459539475908 + }, + "val": { + "rmse": 0.2401098998840954, + "mae": 0.13103342056274414, + "r2": 0.6149019002914429, + "score": -0.2401098998840954 + }, + "test": { + "rmse": 0.24179294965081416, + "mae": 0.14307323098182678, + "r2": 0.6004906892776489, + "score": -0.24179294965081416 + } + }, + "time": "0:02:05.400277", + "diversity": { + "train": { + "mean_centered_corr": -0.01803121715784073, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04816228523850441 + }, + "val": { + "mean_centered_corr": -0.01891929842531681, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06890527904033661 + }, + "test": { + "mean_centered_corr": -0.017366157844662666, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0932842418551445 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015342" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..bfe3860e5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..c0c72f7b4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21942591125322533, + "mae": 0.1180998906493187, + "r2": 0.6923975944519043, + "score": -0.21942591125322533 + }, + "val": { + "rmse": 0.23456885744734607, + "mae": 0.12691855430603027, + "r2": 0.6324707269668579, + "score": -0.23456885744734607 + }, + "test": { + "rmse": 0.23654787235061092, + "mae": 0.15111033618450165, + "r2": 0.6176353693008423, + "score": -0.23654787235061092 + } + }, + "time": "0:02:05.400277", + "diversity": { + "train": { + "mean_centered_corr": -0.01803121715784073, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04816228523850441 + }, + "val": { + "mean_centered_corr": -0.01891929842531681, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06890527904033661 + }, + "test": { + "mean_centered_corr": -0.017366157844662666, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0932842418551445 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2e87bc24c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22062443706127244, + "mae": 0.1181936264038086, + "r2": 0.6890281438827515, + "score": -0.22062443706127244 + }, + "val": { + "rmse": 0.23332136469753356, + "mae": 0.126436248421669, + "r2": 0.636369526386261, + "score": -0.23332136469753356 + }, + "test": { + "rmse": 0.2364625390906921, + "mae": 0.15152248740196228, + "r2": 0.6179112195968628, + "score": -0.2364625390906921 + } + }, + "time": "0:02:05.400277", + "diversity": { + "train": { + "mean_centered_corr": -0.01803121715784073, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04816228523850441 + }, + "val": { + "mean_centered_corr": -0.01891929842531681, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06890527904033661 + }, + "test": { + "mean_centered_corr": -0.017366157844662666, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0932842418551445 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 4, + 21, + 25, + 12, + 16, + 28, + 31, + 22, + 20, + 0, + 25 + ], + "head_selection_time": "0:00:00.189725" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..975a359bd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.23099349784472964, + "mae": 0.13101767003536224, + "r2": 0.6591106653213501, + "score": -0.23099349784472964 + }, + "val": { + "rmse": 0.23704860354665752, + "mae": 0.13406512141227722, + "r2": 0.6246589422225952, + "score": -0.23704860354665752 + }, + "test": { + "rmse": 0.23874426241915847, + "mae": 0.15374092757701874, + "r2": 0.610501766204834, + "score": -0.23874426241915847 + } + }, + "time": "0:01:59.843910", + "diversity": { + "train": { + "mean_centered_corr": -0.018132619559764862, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782529175281525 + }, + "val": { + "mean_centered_corr": -0.018767302855849266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06245988979935646 + }, + "test": { + "mean_centered_corr": -0.016851365566253662, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08885195106267929 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:00.016150" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..5d5322fb9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..0f642a60c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22186950085769538, + "mae": 0.11825541406869888, + "r2": 0.6855083703994751, + "score": -0.22186950085769538 + }, + "val": { + "rmse": 0.23496530422551093, + "mae": 0.12728473544120789, + "r2": 0.6312273144721985, + "score": -0.23496530422551093 + }, + "test": { + "rmse": 0.23600846732737993, + "mae": 0.14936019480228424, + "r2": 0.6193772554397583, + "score": -0.23600846732737993 + } + }, + "time": "0:01:59.843910", + "diversity": { + "train": { + "mean_centered_corr": -0.018132619559764862, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782529175281525 + }, + "val": { + "mean_centered_corr": -0.018767302855849266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06245988979935646 + }, + "test": { + "mean_centered_corr": -0.016851365566253662, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08885195106267929 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..81bbd4812 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,221 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22194279588607752, + "mae": 0.11977781355381012, + "r2": 0.6853005290031433, + "score": -0.22194279588607752 + }, + "val": { + "rmse": 0.23324589552748487, + "mae": 0.12741298973560333, + "r2": 0.6366047263145447, + "score": -0.23324589552748487 + }, + "test": { + "rmse": 0.23661615616090015, + "mae": 0.1510157734155655, + "r2": 0.6174145936965942, + "score": -0.23661615616090015 + } + }, + "time": "0:01:59.843910", + "diversity": { + "train": { + "mean_centered_corr": -0.018132619559764862, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782529175281525 + }, + "val": { + "mean_centered_corr": -0.018767302855849266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06245988979935646 + }, + "test": { + "mean_centered_corr": -0.016851365566253662, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08885195106267929 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 4, + 19, + 22, + 12, + 25, + 30, + 28, + 4 + ], + "head_selection_time": "0:00:00.154365" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e26e031b4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2313707126836422, + "mae": 0.1336175799369812, + "r2": 0.657996416091919, + "score": -0.2313707126836422 + }, + "val": { + "rmse": 0.23835052202476453, + "mae": 0.13680434226989746, + "r2": 0.6205247640609741, + "score": -0.23835052202476453 + }, + "test": { + "rmse": 0.23699550345285816, + "mae": 0.1488882154226303, + "r2": 0.6161868572235107, + "score": -0.23699550345285816 + } + }, + "time": "0:02:00.384580", + "diversity": { + "train": { + "mean_centered_corr": -0.018761252984404564, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04712963104248047 + }, + "val": { + "mean_centered_corr": -0.017407886683940887, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07094229757785797 + }, + "test": { + "mean_centered_corr": -0.01455279067158699, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09494736045598984 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:00.015364" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..fa39890f9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..1d93fe8c8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.222181256913413, + "mae": 0.11818218231201172, + "r2": 0.6846239566802979, + "score": -0.222181256913413 + }, + "val": { + "rmse": 0.23480699780789907, + "mae": 0.12649822235107422, + "r2": 0.6317241191864014, + "score": -0.23480699780789907 + }, + "test": { + "rmse": 0.23801010369895165, + "mae": 0.153095081448555, + "r2": 0.6128935813903809, + "score": -0.23801010369895165 + } + }, + "time": "0:02:00.384580", + "diversity": { + "train": { + "mean_centered_corr": -0.018761252984404564, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04712963104248047 + }, + "val": { + "mean_centered_corr": -0.017407886683940887, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07094229757785797 + }, + "test": { + "mean_centered_corr": -0.01455279067158699, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09494736045598984 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d124019f8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,220 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2223846971387422, + "mae": 0.11872012168169022, + "r2": 0.6840461492538452, + "score": -0.2223846971387422 + }, + "val": { + "rmse": 0.2333328522027255, + "mae": 0.12700752913951874, + "r2": 0.636333703994751, + "score": -0.2333328522027255 + }, + "test": { + "rmse": 0.238922935773872, + "mae": 0.1549610048532486, + "r2": 0.6099185943603516, + "score": -0.238922935773872 + } + }, + "time": "0:02:00.384580", + "diversity": { + "train": { + "mean_centered_corr": -0.018761252984404564, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04712963104248047 + }, + "val": { + "mean_centered_corr": -0.017407886683940887, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07094229757785797 + }, + "test": { + "mean_centered_corr": -0.01455279067158699, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09494736045598984 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 19, + 25, + 12, + 4, + 18, + 31, + 29 + ], + "head_selection_time": "0:00:00.130977" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ed6677fc6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24213441143837863, + "mae": 0.12409187108278275, + "r2": 0.6254352331161499, + "score": -0.24213441143837863 + }, + "val": { + "rmse": 0.23848516306571665, + "mae": 0.13054722547531128, + "r2": 0.6200958490371704, + "score": -0.23848516306571665 + }, + "test": { + "rmse": 0.24223068067706838, + "mae": 0.15242092311382294, + "r2": 0.5990428924560547, + "score": -0.24223068067706838 + } + }, + "time": "0:01:52.170887", + "diversity": { + "train": { + "mean_centered_corr": -0.018726564943790436, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04575931653380394 + }, + "val": { + "mean_centered_corr": -0.019934404641389847, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061195019632577896 + }, + "test": { + "mean_centered_corr": -0.018395286053419113, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08133252710103989 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015563" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..a2e0d9d10 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..72762527c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2329486599171296, + "mae": 0.11878164857625961, + "r2": 0.653315544128418, + "score": -0.2329486599171296 + }, + "val": { + "rmse": 0.2356879335438401, + "mae": 0.12487198412418365, + "r2": 0.6289555430412292, + "score": -0.2356879335438401 + }, + "test": { + "rmse": 0.23192085632684412, + "mae": 0.1407395601272583, + "r2": 0.6324476599693298, + "score": -0.23192085632684412 + } + }, + "time": "0:01:52.170887", + "diversity": { + "train": { + "mean_centered_corr": -0.018726564943790436, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04575931653380394 + }, + "val": { + "mean_centered_corr": -0.019934404641389847, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061195019632577896 + }, + "test": { + "mean_centered_corr": -0.018395286053419113, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08133252710103989 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c8381a924 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,223 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23254022069636512, + "mae": 0.12052810937166214, + "r2": 0.6545302271842957, + "score": -0.23254022069636512 + }, + "val": { + "rmse": 0.23318228828211082, + "mae": 0.12569577991962433, + "r2": 0.6368029117584229, + "score": -0.23318228828211082 + }, + "test": { + "rmse": 0.23425661912073115, + "mae": 0.14765594899654388, + "r2": 0.6250068545341492, + "score": -0.23425661912073115 + } + }, + "time": "0:01:52.170887", + "diversity": { + "train": { + "mean_centered_corr": -0.018726564943790436, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04575931653380394 + }, + "val": { + "mean_centered_corr": -0.019934404641389847, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061195019632577896 + }, + "test": { + "mean_centered_corr": -0.018395286053419113, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08133252710103989 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 20, + 12, + 21, + 25, + 19, + 20, + 0, + 24, + 16 + ], + "head_selection_time": "0:00:00.177504" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c53576bc5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2385315752740614, + "mae": 0.13105398416519165, + "r2": 0.6364989876747131, + "score": -0.2385315752740614 + }, + "val": { + "rmse": 0.23800908632875115, + "mae": 0.13678862154483795, + "r2": 0.6216111183166504, + "score": -0.23800908632875115 + }, + "test": { + "rmse": 0.2395995521584212, + "mae": 0.15617705881595612, + "r2": 0.6077060699462891, + "score": -0.2395995521584212 + } + }, + "time": "0:01:51.161126", + "diversity": { + "train": { + "mean_centered_corr": -0.01903150975704193, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04610341414809227 + }, + "val": { + "mean_centered_corr": -0.0190847497433424, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06286956369876862 + }, + "test": { + "mean_centered_corr": -0.01705675758421421, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08540277928113937 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:00.015566" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..fd91e5498 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..7232e98aa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23162405942251374, + "mae": 0.11943608522415161, + "r2": 0.6572470664978027, + "score": -0.23162405942251374 + }, + "val": { + "rmse": 0.2352761982909868, + "mae": 0.1248830109834671, + "r2": 0.6302508115768433, + "score": -0.2352761982909868 + }, + "test": { + "rmse": 0.2333683170052332, + "mae": 0.14440852403640747, + "r2": 0.6278454065322876, + "score": -0.2333683170052332 + } + }, + "time": "0:01:51.161126", + "diversity": { + "train": { + "mean_centered_corr": -0.01903150975704193, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04610341414809227 + }, + "val": { + "mean_centered_corr": -0.0190847497433424, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06286956369876862 + }, + "test": { + "mean_centered_corr": -0.01705675758421421, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08540277928113937 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..21677580e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,220 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p0001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23244876064945358, + "mae": 0.12024175375699997, + "r2": 0.6548019647598267, + "score": -0.23244876064945358 + }, + "val": { + "rmse": 0.2331158511435021, + "mae": 0.12569448351860046, + "r2": 0.637009859085083, + "score": -0.2331158511435021 + }, + "test": { + "rmse": 0.23340534056347034, + "mae": 0.14711007475852966, + "r2": 0.6277272701263428, + "score": -0.23340534056347034 + } + }, + "time": "0:01:51.161126", + "diversity": { + "train": { + "mean_centered_corr": -0.01903150975704193, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04610341414809227 + }, + "val": { + "mean_centered_corr": -0.0190847497433424, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06286956369876862 + }, + "test": { + "mean_centered_corr": -0.01705675758421421, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08540277928113937 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 28, + 19, + 4, + 18, + 1, + 25, + 20 + ], + "head_selection_time": "0:00:00.131487" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b148197ca --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23600330571914405, + "mae": 0.13268455862998962, + "r2": 0.6441638469696045, + "score": -0.23600330571914405 + }, + "val": { + "rmse": 0.23803283687571922, + "mae": 0.1352454274892807, + "r2": 0.6215356588363647, + "score": -0.23803283687571922 + }, + "test": { + "rmse": 0.273329277420384, + "mae": 0.1970231831073761, + "r2": 0.48948097229003906, + "score": -0.273329277420384 + } + }, + "time": "0:01:51.858305", + "diversity": { + "train": { + "mean_centered_corr": -0.01871478743851185, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046988409012556076 + }, + "val": { + "mean_centered_corr": -0.019513292238116264, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06265565007925034 + }, + "test": { + "mean_centered_corr": -0.018462633714079857, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08088342845439911 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015035" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..a2a06e681 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..ca5cac233 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23085188135419868, + "mae": 0.12120585143566132, + "r2": 0.6595284938812256, + "score": -0.23085188135419868 + }, + "val": { + "rmse": 0.23478980709452008, + "mae": 0.12477234750986099, + "r2": 0.6317780017852783, + "score": -0.23478980709452008 + }, + "test": { + "rmse": 0.2332225120575295, + "mae": 0.1445438116788864, + "r2": 0.6283103227615356, + "score": -0.2332225120575295 + } + }, + "time": "0:01:51.858305", + "diversity": { + "train": { + "mean_centered_corr": -0.01871478743851185, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046988409012556076 + }, + "val": { + "mean_centered_corr": -0.019513292238116264, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06265565007925034 + }, + "test": { + "mean_centered_corr": -0.018462633714079857, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08088342845439911 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f4dda662e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,220 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23141464802856693, + "mae": 0.12309345602989197, + "r2": 0.6578664779663086, + "score": -0.23141464802856693 + }, + "val": { + "rmse": 0.23289638454580067, + "mae": 0.12518544495105743, + "r2": 0.6376929879188538, + "score": -0.23289638454580067 + }, + "test": { + "rmse": 0.23281509730791192, + "mae": 0.1455666571855545, + "r2": 0.6296077966690063, + "score": -0.23281509730791192 + } + }, + "time": "0:01:51.858305", + "diversity": { + "train": { + "mean_centered_corr": -0.01871478743851185, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046988409012556076 + }, + "val": { + "mean_centered_corr": -0.019513292238116264, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06265565007925034 + }, + "test": { + "mean_centered_corr": -0.018462633714079857, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08088342845439911 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 19, + 4, + 16, + 12, + 18, + 1, + 20 + ], + "head_selection_time": "0:00:00.129381" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..96de473ea --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22534824415842816, + "mae": 0.12385565787553787, + "r2": 0.6755690574645996, + "score": -0.22534824415842816 + }, + "val": { + "rmse": 0.24035605783658937, + "mae": 0.13962619006633759, + "r2": 0.6141119003295898, + "score": -0.24035605783658937 + }, + "test": { + "rmse": 0.2779356858657707, + "mae": 0.19685637950897217, + "r2": 0.4721285104751587, + "score": -0.2779356858657707 + } + }, + "time": "0:02:00.459838", + "diversity": { + "train": { + "mean_centered_corr": -0.018568821251392365, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047292761504650116 + }, + "val": { + "mean_centered_corr": -0.01956503838300705, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07274748384952545 + }, + "test": { + "mean_centered_corr": -0.01600496470928192, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1011786088347435 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015299" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..e261269fd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..ab224775e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22215539246899518, + "mae": 0.11718897521495819, + "r2": 0.6846973896026611, + "score": -0.22215539246899518 + }, + "val": { + "rmse": 0.23527326904017234, + "mae": 0.12683525681495667, + "r2": 0.6302599906921387, + "score": -0.23527326904017234 + }, + "test": { + "rmse": 0.24074289728147005, + "mae": 0.15624843537807465, + "r2": 0.6039531230926514, + "score": -0.24074289728147005 + } + }, + "time": "0:02:00.459838", + "diversity": { + "train": { + "mean_centered_corr": -0.018568821251392365, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047292761504650116 + }, + "val": { + "mean_centered_corr": -0.01956503838300705, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07274748384952545 + }, + "test": { + "mean_centered_corr": -0.01600496470928192, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1011786088347435 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..6ae6be381 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,218 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22264168507638651, + "mae": 0.11877916008234024, + "r2": 0.6833154559135437, + "score": -0.22264168507638651 + }, + "val": { + "rmse": 0.23367927924707155, + "mae": 0.12780483067035675, + "r2": 0.6352530717849731, + "score": -0.23367927924707155 + }, + "test": { + "rmse": 0.24524323007832846, + "mae": 0.16379818320274353, + "r2": 0.5890077352523804, + "score": -0.24524323007832846 + } + }, + "time": "0:02:00.459838", + "diversity": { + "train": { + "mean_centered_corr": -0.018568821251392365, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047292761504650116 + }, + "val": { + "mean_centered_corr": -0.01956503838300705, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07274748384952545 + }, + "test": { + "mean_centered_corr": -0.01600496470928192, + "mean_pairwise_disagreement": NaN, + "member_std": 0.1011786088347435 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 19, + 1, + 12, + 29, + 25 + ], + "head_selection_time": "0:00:00.101612" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..174b506e1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22760798675130503, + "mae": 0.12891143560409546, + "r2": 0.6690298318862915, + "score": -0.22760798675130503 + }, + "val": { + "rmse": 0.23789169956459427, + "mae": 0.13463006913661957, + "r2": 0.6219843029975891, + "score": -0.23789169956459427 + }, + "test": { + "rmse": 0.23950414627424274, + "mae": 0.14432719349861145, + "r2": 0.6080183982849121, + "score": -0.23950414627424274 + } + }, + "time": "0:02:00.788746", + "diversity": { + "train": { + "mean_centered_corr": -0.01867086812853813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04923015832901001 + }, + "val": { + "mean_centered_corr": -0.019777491688728333, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06746035814285278 + }, + "test": { + "mean_centered_corr": -0.014284917153418064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09343798458576202 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015396" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..49d9094ee --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..ea35c38ac --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22229246068030295, + "mae": 0.12095138430595398, + "r2": 0.6843081712722778, + "score": -0.22229246068030295 + }, + "val": { + "rmse": 0.23507535721736414, + "mae": 0.1287076324224472, + "r2": 0.6308817863464355, + "score": -0.23507535721736414 + }, + "test": { + "rmse": 0.2433693260742304, + "mae": 0.1609274446964264, + "r2": 0.5952645540237427, + "score": -0.2433693260742304 + } + }, + "time": "0:02:00.788746", + "diversity": { + "train": { + "mean_centered_corr": -0.01867086812853813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04923015832901001 + }, + "val": { + "mean_centered_corr": -0.019777491688728333, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06746035814285278 + }, + "test": { + "mean_centered_corr": -0.014284917153418064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09343798458576202 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..76e167010 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,221 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2227845825987346, + "mae": 0.12195254862308502, + "r2": 0.682908833026886, + "score": -0.2227845825987346 + }, + "val": { + "rmse": 0.23367845823808855, + "mae": 0.1281619518995285, + "r2": 0.6352556347846985, + "score": -0.23367845823808855 + }, + "test": { + "rmse": 0.24371457881808845, + "mae": 0.1601243019104004, + "r2": 0.5941153764724731, + "score": -0.24371457881808845 + } + }, + "time": "0:02:00.788746", + "diversity": { + "train": { + "mean_centered_corr": -0.01867086812853813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04923015832901001 + }, + "val": { + "mean_centered_corr": -0.019777491688728333, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06746035814285278 + }, + "test": { + "mean_centered_corr": -0.014284917153418064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09343798458576202 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 12, + 28, + 25, + 8, + 4, + 12, + 16, + 17 + ], + "head_selection_time": "0:00:00.145937" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..69abc5cba --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23691756457376573, + "mae": 0.130058154463768, + "r2": 0.6414015293121338, + "score": -0.23691756457376573 + }, + "val": { + "rmse": 0.23784205343302714, + "mae": 0.13620039820671082, + "r2": 0.6221420764923096, + "score": -0.23784205343302714 + }, + "test": { + "rmse": 0.23492100218498793, + "mae": 0.14837363362312317, + "r2": 0.622876763343811, + "score": -0.23492100218498793 + } + }, + "time": "0:01:51.789439", + "diversity": { + "train": { + "mean_centered_corr": -0.018733546137809753, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04638450965285301 + }, + "val": { + "mean_centered_corr": -0.019742805510759354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060568567365407944 + }, + "test": { + "mean_centered_corr": -0.018700512126088142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.078513003885746 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:00.015130" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..180a4eb3d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..cae422be4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23100547201881677, + "mae": 0.12135384976863861, + "r2": 0.6590753197669983, + "score": -0.23100547201881677 + }, + "val": { + "rmse": 0.23446964895916758, + "mae": 0.1270645558834076, + "r2": 0.6327815055847168, + "score": -0.23446964895916758 + }, + "test": { + "rmse": 0.23505021426230255, + "mae": 0.1497301310300827, + "r2": 0.6224617958068848, + "score": -0.23505021426230255 + } + }, + "time": "0:01:51.789439", + "diversity": { + "train": { + "mean_centered_corr": -0.018733546137809753, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04638450965285301 + }, + "val": { + "mean_centered_corr": -0.019742805510759354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060568567365407944 + }, + "test": { + "mean_centered_corr": -0.018700512126088142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.078513003885746 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..a39bb4840 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,223 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n0p001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23208180560739772, + "mae": 0.12161051481962204, + "r2": 0.655890941619873, + "score": -0.23208180560739772 + }, + "val": { + "rmse": 0.23278002032353212, + "mae": 0.1258585900068283, + "r2": 0.6380549669265747, + "score": -0.23278002032353212 + }, + "test": { + "rmse": 0.23227273339288673, + "mae": 0.14668183028697968, + "r2": 0.6313314437866211, + "score": -0.23227273339288673 + } + }, + "time": "0:01:51.789439", + "diversity": { + "train": { + "mean_centered_corr": -0.018733546137809753, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04638450965285301 + }, + "val": { + "mean_centered_corr": -0.019742805510759354, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060568567365407944 + }, + "test": { + "mean_centered_corr": -0.018700512126088142, + "mean_pairwise_disagreement": NaN, + "member_std": 0.078513003885746 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 28, + 4, + 12, + 20, + 21, + 0, + 24, + 4, + 1, + 19 + ], + "head_selection_time": "0:00:00.172089" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..024b13351 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23738592821530582, + "mae": 0.12199530750513077, + "r2": 0.6399823427200317, + "score": -0.23738592821530582 + }, + "val": { + "rmse": 0.23736952064793038, + "mae": 0.12993131577968597, + "r2": 0.6236419677734375, + "score": -0.23736952064793038 + }, + "test": { + "rmse": 0.24275107338262392, + "mae": 0.15340644121170044, + "r2": 0.5973182916641235, + "score": -0.24275107338262392 + } + }, + "time": "0:01:52.147419", + "diversity": { + "train": { + "mean_centered_corr": -0.018815485760569572, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046015676110982895 + }, + "val": { + "mean_centered_corr": -0.018479591235518456, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06330302357673645 + }, + "test": { + "mean_centered_corr": -0.01912987418472767, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08374166488647461 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015222" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..423740a5b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..12a1c544e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23060420468503218, + "mae": 0.11958978325128555, + "r2": 0.6602587103843689, + "score": -0.23060420468503218 + }, + "val": { + "rmse": 0.23516112255992166, + "mae": 0.12509571015834808, + "r2": 0.6306123733520508, + "score": -0.23516112255992166 + }, + "test": { + "rmse": 0.23211789488202353, + "mae": 0.14202691614627838, + "r2": 0.6318228244781494, + "score": -0.23211789488202353 + } + }, + "time": "0:01:52.147419", + "diversity": { + "train": { + "mean_centered_corr": -0.018815485760569572, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046015676110982895 + }, + "val": { + "mean_centered_corr": -0.018479591235518456, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06330302357673645 + }, + "test": { + "mean_centered_corr": -0.01912987418472767, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08374166488647461 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..44baedc4e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23160982522236812, + "mae": 0.12151593714952469, + "r2": 0.6572891473770142, + "score": -0.23160982522236812 + }, + "val": { + "rmse": 0.23322604208950684, + "mae": 0.12618425488471985, + "r2": 0.6366665959358215, + "score": -0.23322604208950684 + }, + "test": { + "rmse": 0.23242508060584816, + "mae": 0.1440695971250534, + "r2": 0.630847692489624, + "score": -0.23242508060584816 + } + }, + "time": "0:01:52.147419", + "diversity": { + "train": { + "mean_centered_corr": -0.018815485760569572, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046015676110982895 + }, + "val": { + "mean_centered_corr": -0.018479591235518456, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06330302357673645 + }, + "test": { + "mean_centered_corr": -0.01912987418472767, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08374166488647461 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 16, + 19, + 0, + 21, + 16, + 12 + ], + "head_selection_time": "0:00:00.114905" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c7de4a402 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22784794156293847, + "mae": 0.12659133970737457, + "r2": 0.6683316230773926, + "score": -0.22784794156293847 + }, + "val": { + "rmse": 0.2393372540058719, + "mae": 0.1392020285129547, + "r2": 0.6173763275146484, + "score": -0.2393372540058719 + }, + "test": { + "rmse": 0.2906284957593463, + "mae": 0.21522651612758636, + "r2": 0.4228137731552124, + "score": -0.2906284957593463 + } + }, + "time": "0:01:59.475961", + "diversity": { + "train": { + "mean_centered_corr": -0.019331008195877075, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04783523082733154 + }, + "val": { + "mean_centered_corr": -0.019431548193097115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06572002172470093 + }, + "test": { + "mean_centered_corr": -0.016879823058843613, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08989977091550827 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015390" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..8c97c9fd2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..a9d12d5ce --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22281919339573966, + "mae": 0.11718059331178665, + "r2": 0.6828103065490723, + "score": -0.22281919339573966 + }, + "val": { + "rmse": 0.23464263099787872, + "mae": 0.12664183974266052, + "r2": 0.6322394609451294, + "score": -0.23464263099787872 + }, + "test": { + "rmse": 0.23482863711969398, + "mae": 0.14775922894477844, + "r2": 0.6231732368469238, + "score": -0.23482863711969398 + } + }, + "time": "0:01:59.475961", + "diversity": { + "train": { + "mean_centered_corr": -0.019331008195877075, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04783523082733154 + }, + "val": { + "mean_centered_corr": -0.019431548193097115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06572002172470093 + }, + "test": { + "mean_centered_corr": -0.016879823058843613, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08989977091550827 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..e9bacf5cd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22351819228584932, + "mae": 0.11853836476802826, + "r2": 0.6808170676231384, + "score": -0.22351819228584932 + }, + "val": { + "rmse": 0.23272791516423366, + "mae": 0.1259242445230484, + "r2": 0.6382169723510742, + "score": -0.23272791516423366 + }, + "test": { + "rmse": 0.23258942887652473, + "mae": 0.14525124430656433, + "r2": 0.630325436592102, + "score": -0.23258942887652473 + } + }, + "time": "0:01:59.475961", + "diversity": { + "train": { + "mean_centered_corr": -0.019331008195877075, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04783523082733154 + }, + "val": { + "mean_centered_corr": -0.019431548193097115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06572002172470093 + }, + "test": { + "mean_centered_corr": -0.016879823058843613, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08989977091550827 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 12, + 16, + 19, + 4, + 29, + 12, + 28, + 20, + 21, + 19, + 16, + 12 + ], + "head_selection_time": "0:00:00.204657" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5173871f0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23885602106177764, + "mae": 0.12304521352052689, + "r2": 0.6355094313621521, + "score": -0.23885602106177764 + }, + "val": { + "rmse": 0.23826622524732466, + "mae": 0.13159264624118805, + "r2": 0.6207931041717529, + "score": -0.23826622524732466 + }, + "test": { + "rmse": 0.2354789879616611, + "mae": 0.14298199117183685, + "r2": 0.62108314037323, + "score": -0.2354789879616611 + } + }, + "time": "0:01:51.731542", + "diversity": { + "train": { + "mean_centered_corr": -0.01938508450984955, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04722513630986214 + }, + "val": { + "mean_centered_corr": -0.01980144903063774, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06155725568532944 + }, + "test": { + "mean_centered_corr": -0.01934674195945263, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08168060332536697 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015102" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..dd37e163a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..1067761ef --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23113198136255483, + "mae": 0.1195804625749588, + "r2": 0.6587017774581909, + "score": -0.23113198136255483 + }, + "val": { + "rmse": 0.23461772747586743, + "mae": 0.12575800716876984, + "r2": 0.6323175430297852, + "score": -0.23461772747586743 + }, + "test": { + "rmse": 0.23292056846137055, + "mae": 0.14504367113113403, + "r2": 0.6292721033096313, + "score": -0.23292056846137055 + } + }, + "time": "0:01:51.731542", + "diversity": { + "train": { + "mean_centered_corr": -0.01938508450984955, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04722513630986214 + }, + "val": { + "mean_centered_corr": -0.01980144903063774, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06155725568532944 + }, + "test": { + "mean_centered_corr": -0.01934674195945263, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08168060332536697 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..55de1b3f6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,223 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23143950990251744, + "mae": 0.11987357586622238, + "r2": 0.6577929854393005, + "score": -0.23143950990251744 + }, + "val": { + "rmse": 0.23311639447730767, + "mae": 0.12623491883277893, + "r2": 0.6370081305503845, + "score": -0.23311639447730767 + }, + "test": { + "rmse": 0.2316950082866651, + "mae": 0.1445348560810089, + "r2": 0.6331631541252136, + "score": -0.2316950082866651 + } + }, + "time": "0:01:51.731542", + "diversity": { + "train": { + "mean_centered_corr": -0.01938508450984955, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04722513630986214 + }, + "val": { + "mean_centered_corr": -0.01980144903063774, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06155725568532944 + }, + "test": { + "mean_centered_corr": -0.01934674195945263, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08168060332536697 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 19, + 18, + 20, + 0, + 28, + 1, + 21, + 12, + 16, + 20 + ], + "head_selection_time": "0:00:00.172745" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f5cbf59f1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24032697986190826, + "mae": 0.12809138000011444, + "r2": 0.6310063004493713, + "score": -0.24032697986190826 + }, + "val": { + "rmse": 0.23826509952531, + "mae": 0.13267768919467926, + "r2": 0.6207966804504395, + "score": -0.23826509952531 + }, + "test": { + "rmse": 0.23646242093377673, + "mae": 0.14509999752044678, + "r2": 0.6179115772247314, + "score": -0.23646242093377673 + } + }, + "time": "0:01:52.868935", + "diversity": { + "train": { + "mean_centered_corr": -0.018483418971300125, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04600879177451134 + }, + "val": { + "mean_centered_corr": -0.019203318282961845, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06106279790401459 + }, + "test": { + "mean_centered_corr": -0.016553888097405434, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07980619370937347 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015166" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..e12a2c399 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..a8928f6bd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23176596757874662, + "mae": 0.12220603227615356, + "r2": 0.656826913356781, + "score": -0.23176596757874662 + }, + "val": { + "rmse": 0.23469026342966967, + "mae": 0.126294806599617, + "r2": 0.6320902109146118, + "score": -0.23469026342966967 + }, + "test": { + "rmse": 0.23333669987162928, + "mae": 0.1461082249879837, + "r2": 0.6279462575912476, + "score": -0.23333669987162928 + } + }, + "time": "0:01:52.868935", + "diversity": { + "train": { + "mean_centered_corr": -0.018483418971300125, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04600879177451134 + }, + "val": { + "mean_centered_corr": -0.019203318282961845, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06106279790401459 + }, + "test": { + "mean_centered_corr": -0.016553888097405434, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07980619370937347 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..646d15121 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r2_n1em05_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23300054783459956, + "mae": 0.12348049134016037, + "r2": 0.6531611680984497, + "score": -0.23300054783459956 + }, + "val": { + "rmse": 0.23303048406462032, + "mae": 0.12674884498119354, + "r2": 0.6372756361961365, + "score": -0.23303048406462032 + }, + "test": { + "rmse": 0.23273007610764737, + "mae": 0.14747364819049835, + "r2": 0.6298782229423523, + "score": -0.23273007610764737 + } + }, + "time": "0:01:52.868935", + "diversity": { + "train": { + "mean_centered_corr": -0.018483418971300125, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04600879177451134 + }, + "val": { + "mean_centered_corr": -0.019203318282961845, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06106279790401459 + }, + "test": { + "mean_centered_corr": -0.016553888097405434, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07980619370937347 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 20, + 0, + 28, + 1, + 12, + 16 + ], + "head_selection_time": "0:00:00.114933" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..350fafda8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2369720026477417, + "mae": 0.12161877006292343, + "r2": 0.6412367224693298, + "score": -0.2369720026477417 + }, + "val": { + "rmse": 0.23818537080734506, + "mae": 0.13046246767044067, + "r2": 0.6210504174232483, + "score": -0.23818537080734506 + }, + "test": { + "rmse": 0.23894844295090592, + "mae": 0.1457148939371109, + "r2": 0.6098352670669556, + "score": -0.23894844295090592 + } + }, + "time": "0:02:41.714436", + "diversity": { + "train": { + "mean_centered_corr": -0.01929367519915104, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047684069722890854 + }, + "val": { + "mean_centered_corr": -0.01931084133684635, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06489446014165878 + }, + "test": { + "mean_centered_corr": -0.016153957694768906, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0877329483628273 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015071" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..9720c0f9b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..396b8b8e0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23113544661823018, + "mae": 0.12141359597444534, + "r2": 0.6586915850639343, + "score": -0.23113544661823018 + }, + "val": { + "rmse": 0.23541561220714985, + "mae": 0.12720422446727753, + "r2": 0.629812479019165, + "score": -0.23541561220714985 + }, + "test": { + "rmse": 0.23578183426225294, + "mae": 0.14981894195079803, + "r2": 0.620107889175415, + "score": -0.23578183426225294 + } + }, + "time": "0:02:41.714436", + "diversity": { + "train": { + "mean_centered_corr": -0.01929367519915104, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047684069722890854 + }, + "val": { + "mean_centered_corr": -0.01931084133684635, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06489446014165878 + }, + "test": { + "mean_centered_corr": -0.016153957694768906, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0877329483628273 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d0c204b9b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,220 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23155468155329886, + "mae": 0.12149085104465485, + "r2": 0.6574523448944092, + "score": -0.23155468155329886 + }, + "val": { + "rmse": 0.2337721462391934, + "mae": 0.1271917074918747, + "r2": 0.6349630951881409, + "score": -0.2337721462391934 + }, + "test": { + "rmse": 0.2337674053617211, + "mae": 0.1486269235610962, + "r2": 0.6265714168548584, + "score": -0.2337674053617211 + } + }, + "time": "0:02:41.714436", + "diversity": { + "train": { + "mean_centered_corr": -0.01929367519915104, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047684069722890854 + }, + "val": { + "mean_centered_corr": -0.01931084133684635, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06489446014165878 + }, + "test": { + "mean_centered_corr": -0.016153957694768906, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0877329483628273 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 28, + 20, + 18, + 1, + 29, + 16 + ], + "head_selection_time": "0:00:00.128427" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6e119ee90 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2384340312375474, + "mae": 0.12352686375379562, + "r2": 0.636796236038208, + "score": -0.2384340312375474 + }, + "val": { + "rmse": 0.23826995416351013, + "mae": 0.13112401962280273, + "r2": 0.6207812428474426, + "score": -0.23826995416351013 + }, + "test": { + "rmse": 0.23722919786139868, + "mae": 0.13967889547348022, + "r2": 0.6154295802116394, + "score": -0.23722919786139868 + } + }, + "time": "0:02:42.229099", + "diversity": { + "train": { + "mean_centered_corr": -0.019007209688425064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04895399883389473 + }, + "val": { + "mean_centered_corr": -0.01975170709192753, + "mean_pairwise_disagreement": NaN, + "member_std": 0.065635085105896 + }, + "test": { + "mean_centered_corr": -0.01898389309644699, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08565043658018112 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015367" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..70afd9cf8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..00d884281 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2313063804097323, + "mae": 0.123744435608387, + "r2": 0.6581865549087524, + "score": -0.2313063804097323 + }, + "val": { + "rmse": 0.23501861733133902, + "mae": 0.12678450345993042, + "r2": 0.631060004234314, + "score": -0.23501861733133902 + }, + "test": { + "rmse": 0.23373303710107815, + "mae": 0.14622008800506592, + "r2": 0.6266812086105347, + "score": -0.23373303710107815 + } + }, + "time": "0:02:42.229099", + "diversity": { + "train": { + "mean_centered_corr": -0.019007209688425064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04895399883389473 + }, + "val": { + "mean_centered_corr": -0.01975170709192753, + "mean_pairwise_disagreement": NaN, + "member_std": 0.065635085105896 + }, + "test": { + "mean_centered_corr": -0.01898389309644699, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08565043658018112 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..69116c88f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,218 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23205218044116352, + "mae": 0.12285098433494568, + "r2": 0.655978798866272, + "score": -0.23205218044116352 + }, + "val": { + "rmse": 0.23341469329024547, + "mae": 0.12756626307964325, + "r2": 0.6360785961151123, + "score": -0.23341469329024547 + }, + "test": { + "rmse": 0.23271509315390915, + "mae": 0.14598432183265686, + "r2": 0.6299258470535278, + "score": -0.23271509315390915 + } + }, + "time": "0:02:42.229099", + "diversity": { + "train": { + "mean_centered_corr": -0.019007209688425064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04895399883389473 + }, + "val": { + "mean_centered_corr": -0.01975170709192753, + "mean_pairwise_disagreement": NaN, + "member_std": 0.065635085105896 + }, + "test": { + "mean_centered_corr": -0.01898389309644699, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08565043658018112 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 1, + 18, + 0, + 20, + 21 + ], + "head_selection_time": "0:00:00.102363" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..11e10d745 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22814835272103823, + "mae": 0.12647606432437897, + "r2": 0.6674563884735107, + "score": -0.22814835272103823 + }, + "val": { + "rmse": 0.23860257792551076, + "mae": 0.1318465769290924, + "r2": 0.6197217106819153, + "score": -0.23860257792551076 + }, + "test": { + "rmse": 0.30043019206533295, + "mae": 0.22363102436065674, + "r2": 0.3832250237464905, + "score": -0.30043019206533295 + } + }, + "time": "0:02:55.073822", + "diversity": { + "train": { + "mean_centered_corr": -0.018881838768720627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04876598343253136 + }, + "val": { + "mean_centered_corr": -0.01626105234026909, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07298418134450912 + }, + "test": { + "mean_centered_corr": -0.016811326146125793, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0990188717842102 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015371" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..b75922aea --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..7c20cf7f3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22307667441316298, + "mae": 0.11820665746927261, + "r2": 0.6820768117904663, + "score": -0.22307667441316298 + }, + "val": { + "rmse": 0.23496360777291245, + "mae": 0.12653271853923798, + "r2": 0.6312326192855835, + "score": -0.23496360777291245 + }, + "test": { + "rmse": 0.23914605469445158, + "mae": 0.15438953042030334, + "r2": 0.6091896891593933, + "score": -0.23914605469445158 + } + }, + "time": "0:02:55.073822", + "diversity": { + "train": { + "mean_centered_corr": -0.018881838768720627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04876598343253136 + }, + "val": { + "mean_centered_corr": -0.01626105234026909, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07298418134450912 + }, + "test": { + "mean_centered_corr": -0.016811326146125793, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0990188717842102 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..50acbd003 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2248165916254, + "mae": 0.1193905919790268, + "r2": 0.6770980954170227, + "score": -0.2248165916254 + }, + "val": { + "rmse": 0.2336154232368663, + "mae": 0.124774269759655, + "r2": 0.635452389717102, + "score": -0.2336154232368663 + }, + "test": { + "rmse": 0.24064628828924506, + "mae": 0.1569826304912567, + "r2": 0.6042709350585938, + "score": -0.24064628828924506 + } + }, + "time": "0:02:55.073822", + "diversity": { + "train": { + "mean_centered_corr": -0.018881838768720627, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04876598343253136 + }, + "val": { + "mean_centered_corr": -0.01626105234026909, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07298418134450912 + }, + "test": { + "mean_centered_corr": -0.016811326146125793, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0990188717842102 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 22, + 12, + 4, + 15, + 16, + 8 + ], + "head_selection_time": "0:00:00.116993" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..89dfbd742 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2322114103746433, + "mae": 0.1252126395702362, + "r2": 0.6555064916610718, + "score": -0.2322114103746433 + }, + "val": { + "rmse": 0.23785759049732064, + "mae": 0.13416989147663116, + "r2": 0.6220927238464355, + "score": -0.23785759049732064 + }, + "test": { + "rmse": 0.23589133259357026, + "mae": 0.1475115269422531, + "r2": 0.6197549104690552, + "score": -0.23589133259357026 + } + }, + "time": "0:02:41.937734", + "diversity": { + "train": { + "mean_centered_corr": -0.019183777272701263, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048386890441179276 + }, + "val": { + "mean_centered_corr": -0.018359092995524406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06268003582954407 + }, + "test": { + "mean_centered_corr": -0.017718803137540817, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08186227828264236 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 20 + ], + "head_selection_time": "0:00:00.015353" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..77c7a7bbb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..0b0f41ad0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23080944483078117, + "mae": 0.12102995067834854, + "r2": 0.6596536636352539, + "score": -0.23080944483078117 + }, + "val": { + "rmse": 0.23463987641889242, + "mae": 0.1267199069261551, + "r2": 0.6322481036186218, + "score": -0.23463987641889242 + }, + "test": { + "rmse": 0.23470931052134267, + "mae": 0.14773716032505035, + "r2": 0.6235561370849609, + "score": -0.23470931052134267 + } + }, + "time": "0:02:41.937734", + "diversity": { + "train": { + "mean_centered_corr": -0.019183777272701263, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048386890441179276 + }, + "val": { + "mean_centered_corr": -0.018359092995524406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06268003582954407 + }, + "test": { + "mean_centered_corr": -0.017718803137540817, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08186227828264236 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..21032cc0a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,220 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23172323223879757, + "mae": 0.12236517667770386, + "r2": 0.6569534540176392, + "score": -0.23172323223879757 + }, + "val": { + "rmse": 0.23269480219963232, + "mae": 0.1264098584651947, + "r2": 0.6383199095726013, + "score": -0.23269480219963232 + }, + "test": { + "rmse": 0.23239933834285403, + "mae": 0.1455182433128357, + "r2": 0.6309294700622559, + "score": -0.23239933834285403 + } + }, + "time": "0:02:41.937734", + "diversity": { + "train": { + "mean_centered_corr": -0.019183777272701263, + "mean_pairwise_disagreement": NaN, + "member_std": 0.048386890441179276 + }, + "val": { + "mean_centered_corr": -0.018359092995524406, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06268003582954407 + }, + "test": { + "mean_centered_corr": -0.017718803137540817, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08186227828264236 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 20, + 1, + 4, + 28, + 19, + 21, + 16, + 0 + ], + "head_selection_time": "0:00:00.131351" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8759b300a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22972785536803275, + "mae": 0.12533853948116302, + "r2": 0.662835955619812, + "score": -0.22972785536803275 + }, + "val": { + "rmse": 0.23957945554469914, + "mae": 0.13747090101242065, + "r2": 0.6166014671325684, + "score": -0.23957945554469914 + }, + "test": { + "rmse": 0.2496147390992142, + "mae": 0.1673688292503357, + "r2": 0.574225127696991, + "score": -0.2496147390992142 + } + }, + "time": "0:02:54.258816", + "diversity": { + "train": { + "mean_centered_corr": -0.018549814820289612, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047382600605487823 + }, + "val": { + "mean_centered_corr": -0.019721152260899544, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06767292320728302 + }, + "test": { + "mean_centered_corr": -0.01792355626821518, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0957256406545639 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22 + ], + "head_selection_time": "0:00:00.015165" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..f46785b6a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..74c149180 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22091751319825467, + "mae": 0.11859732866287231, + "r2": 0.688201367855072, + "score": -0.22091751319825467 + }, + "val": { + "rmse": 0.23513454704634784, + "mae": 0.12659163773059845, + "r2": 0.6306958794593811, + "score": -0.23513454704634784 + }, + "test": { + "rmse": 0.2392125690267148, + "mae": 0.15437257289886475, + "r2": 0.6089722514152527, + "score": -0.2392125690267148 + } + }, + "time": "0:02:54.258816", + "diversity": { + "train": { + "mean_centered_corr": -0.018549814820289612, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047382600605487823 + }, + "val": { + "mean_centered_corr": -0.019721152260899544, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06767292320728302 + }, + "test": { + "mean_centered_corr": -0.01792355626821518, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0957256406545639 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..6e296ff89 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,220 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22177797398882887, + "mae": 0.12039296329021454, + "r2": 0.6857677698135376, + "score": -0.22177797398882887 + }, + "val": { + "rmse": 0.23353144315489308, + "mae": 0.1272912323474884, + "r2": 0.6357144117355347, + "score": -0.23353144315489308 + }, + "test": { + "rmse": 0.2478765142940872, + "mae": 0.16908910870552063, + "r2": 0.5801343321800232, + "score": -0.2478765142940872 + } + }, + "time": "0:02:54.258816", + "diversity": { + "train": { + "mean_centered_corr": -0.018549814820289612, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047382600605487823 + }, + "val": { + "mean_centered_corr": -0.019721152260899544, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06767292320728302 + }, + "test": { + "mean_centered_corr": -0.01792355626821518, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0957256406545639 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 22, + 12, + 16, + 27, + 25, + 4, + 20, + 18 + ], + "head_selection_time": "0:00:00.131334" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..2aa5df0cb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23493290303475395, + "mae": 0.13081301748752594, + "r2": 0.6473843455314636, + "score": -0.23493290303475395 + }, + "val": { + "rmse": 0.2382764815807058, + "mae": 0.12931407988071442, + "r2": 0.620760440826416, + "score": -0.2382764815807058 + }, + "test": { + "rmse": 0.24121307650092602, + "mae": 0.14123845100402832, + "r2": 0.6024046540260315, + "score": -0.24121307650092602 + } + }, + "time": "0:02:41.290051", + "diversity": { + "train": { + "mean_centered_corr": -0.01915917731821537, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044233281165361404 + }, + "val": { + "mean_centered_corr": -0.01909327693283558, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06557415425777435 + }, + "test": { + "mean_centered_corr": -0.01792578585445881, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08976518362760544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.018212" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..1892f3234 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..632edb3d9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23079886474485123, + "mae": 0.11978797614574432, + "r2": 0.6596848964691162, + "score": -0.23079886474485123 + }, + "val": { + "rmse": 0.2354849520464804, + "mae": 0.12464786320924759, + "r2": 0.6295943260192871, + "score": -0.2354849520464804 + }, + "test": { + "rmse": 0.23141729611827572, + "mae": 0.13986103236675262, + "r2": 0.6340420246124268, + "score": -0.23141729611827572 + } + }, + "time": "0:02:41.290051", + "diversity": { + "train": { + "mean_centered_corr": -0.01915917731821537, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044233281165361404 + }, + "val": { + "mean_centered_corr": -0.01909327693283558, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06557415425777435 + }, + "test": { + "mean_centered_corr": -0.01792578585445881, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08976518362760544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..92e257186 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,223 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2315166218673027, + "mae": 0.11995172500610352, + "r2": 0.6575649380683899, + "score": -0.2315166218673027 + }, + "val": { + "rmse": 0.23269733966625106, + "mae": 0.12517677247524261, + "r2": 0.6383119821548462, + "score": -0.23269733966625106 + }, + "test": { + "rmse": 0.23483414974881817, + "mae": 0.14970079064369202, + "r2": 0.6231555938720703, + "score": -0.23483414974881817 + } + }, + "time": "0:02:41.290051", + "diversity": { + "train": { + "mean_centered_corr": -0.01915917731821537, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044233281165361404 + }, + "val": { + "mean_centered_corr": -0.01909327693283558, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06557415425777435 + }, + "test": { + "mean_centered_corr": -0.01792578585445881, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08976518362760544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 18, + 12, + 28, + 4, + 25, + 31, + 18, + 21, + 4, + 12 + ], + "head_selection_time": "0:00:00.182946" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4766259c2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22476767062044256, + "mae": 0.1237136721611023, + "r2": 0.6772385835647583, + "score": -0.22476767062044256 + }, + "val": { + "rmse": 0.2387734473283714, + "mae": 0.13085761666297913, + "r2": 0.6191768646240234, + "score": -0.2387734473283714 + }, + "test": { + "rmse": 0.23954579674332094, + "mae": 0.14230574667453766, + "r2": 0.6078820824623108, + "score": -0.23954579674332094 + } + }, + "time": "0:02:55.252644", + "diversity": { + "train": { + "mean_centered_corr": -0.018560394644737244, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04619770124554634 + }, + "val": { + "mean_centered_corr": -0.01867174729704857, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06765128672122955 + }, + "test": { + "mean_centered_corr": -0.01507267914712429, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09221170097589493 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015386" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..7097b4f27 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..054e40dad --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22135730441992338, + "mae": 0.11825704574584961, + "r2": 0.6869587302207947, + "score": -0.22135730441992338 + }, + "val": { + "rmse": 0.23554663270650014, + "mae": 0.1282879263162613, + "r2": 0.6294003129005432, + "score": -0.23554663270650014 + }, + "test": { + "rmse": 0.24084128499653581, + "mae": 0.15699797868728638, + "r2": 0.6036293506622314, + "score": -0.24084128499653581 + } + }, + "time": "0:02:55.252644", + "diversity": { + "train": { + "mean_centered_corr": -0.018560394644737244, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04619770124554634 + }, + "val": { + "mean_centered_corr": -0.01867174729704857, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06765128672122955 + }, + "test": { + "mean_centered_corr": -0.01507267914712429, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09221170097589493 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..e953f4f28 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,225 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2225935831054593, + "mae": 0.11814232915639877, + "r2": 0.6834522485733032, + "score": -0.2225935831054593 + }, + "val": { + "rmse": 0.23413401771618858, + "mae": 0.12737752497196198, + "r2": 0.6338320970535278, + "score": -0.23413401771618858 + }, + "test": { + "rmse": 0.23885206734563352, + "mae": 0.15390992164611816, + "r2": 0.6101499795913696, + "score": -0.23885206734563352 + } + }, + "time": "0:02:55.252644", + "diversity": { + "train": { + "mean_centered_corr": -0.018560394644737244, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04619770124554634 + }, + "val": { + "mean_centered_corr": -0.01867174729704857, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06765128672122955 + }, + "test": { + "mean_centered_corr": -0.01507267914712429, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09221170097589493 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 22, + 18, + 4, + 27, + 8, + 17, + 12, + 16, + 29, + 20, + 21, + 22 + ], + "head_selection_time": "0:00:00.204447" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c2d6fb60b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 740, + "metrics": { + "train": { + "rmse": 0.23212312683947872, + "mae": 0.12872183322906494, + "r2": 0.6557683944702148, + "score": -0.23212312683947872 + }, + "val": { + "rmse": 0.2393572230996531, + "mae": 0.13352128863334656, + "r2": 0.6173124313354492, + "score": -0.2393572230996531 + }, + "test": { + "rmse": 0.23868415725996342, + "mae": 0.1462382674217224, + "r2": 0.610697865486145, + "score": -0.23868415725996342 + } + }, + "time": "0:02:49.580435", + "diversity": { + "train": { + "mean_centered_corr": -0.01846710592508316, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04601765796542168 + }, + "val": { + "mean_centered_corr": -0.018632514402270317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06924604624509811 + }, + "test": { + "mean_centered_corr": -0.017972121015191078, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09207665920257568 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015228" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..13ab81f6d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..3fd44c48f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 740, + "metrics": { + "train": { + "rmse": 0.22632894316351912, + "mae": 0.1234244555234909, + "r2": 0.6727391481399536, + "score": -0.22632894316351912 + }, + "val": { + "rmse": 0.23546283513255054, + "mae": 0.12725704908370972, + "r2": 0.6296639442443848, + "score": -0.23546283513255054 + }, + "test": { + "rmse": 0.23448563984262677, + "mae": 0.14775648713111877, + "r2": 0.6242733001708984, + "score": -0.23448563984262677 + } + }, + "time": "0:02:49.580435", + "diversity": { + "train": { + "mean_centered_corr": -0.01846710592508316, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04601765796542168 + }, + "val": { + "mean_centered_corr": -0.018632514402270317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06924604624509811 + }, + "test": { + "mean_centered_corr": -0.017972121015191078, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09207665920257568 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..23ff3622f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,221 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p0001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 740, + "metrics": { + "train": { + "rmse": 0.22564756699293684, + "mae": 0.12296506017446518, + "r2": 0.6747066378593445, + "score": -0.22564756699293684 + }, + "val": { + "rmse": 0.23411584672739014, + "mae": 0.12701718509197235, + "r2": 0.6338889598846436, + "score": -0.23411584672739014 + }, + "test": { + "rmse": 0.23627903325948496, + "mae": 0.15204240381717682, + "r2": 0.6185040473937988, + "score": -0.23627903325948496 + } + }, + "time": "0:02:49.580435", + "diversity": { + "train": { + "mean_centered_corr": -0.01846710592508316, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04601765796542168 + }, + "val": { + "mean_centered_corr": -0.018632514402270317, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06924604624509811 + }, + "test": { + "mean_centered_corr": -0.017972121015191078, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09207665920257568 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 28, + 12, + 20, + 18, + 25, + 4, + 5, + 12 + ], + "head_selection_time": "0:00:00.145152" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..627b71fc0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22625887190827382, + "mae": 0.12710008025169373, + "r2": 0.6729417443275452, + "score": -0.22625887190827382 + }, + "val": { + "rmse": 0.23889975711739106, + "mae": 0.13583312928676605, + "r2": 0.6187738180160522, + "score": -0.23889975711739106 + }, + "test": { + "rmse": 0.23704205803317221, + "mae": 0.14239075779914856, + "r2": 0.616036057472229, + "score": -0.23704205803317221 + } + }, + "time": "0:02:52.457107", + "diversity": { + "train": { + "mean_centered_corr": -0.01820620521903038, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04663686826825142 + }, + "val": { + "mean_centered_corr": -0.01946132257580757, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06672783941030502 + }, + "test": { + "mean_centered_corr": -0.017525583505630493, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09117421507835388 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015231" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..3bc8abdac --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..0b69654e8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22173162515690617, + "mae": 0.11826841533184052, + "r2": 0.6858991384506226, + "score": -0.22173162515690617 + }, + "val": { + "rmse": 0.234735553294385, + "mae": 0.1262959986925125, + "r2": 0.6319481134414673, + "score": -0.234735553294385 + }, + "test": { + "rmse": 0.23656269916289197, + "mae": 0.15081582963466644, + "r2": 0.6175874471664429, + "score": -0.23656269916289197 + } + }, + "time": "0:02:52.457107", + "diversity": { + "train": { + "mean_centered_corr": -0.01820620521903038, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04663686826825142 + }, + "val": { + "mean_centered_corr": -0.01946132257580757, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06672783941030502 + }, + "test": { + "mean_centered_corr": -0.017525583505630493, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09117421507835388 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..bce2773ab --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,226 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22176081479492696, + "mae": 0.11819849163293839, + "r2": 0.6858164072036743, + "score": -0.22176081479492696 + }, + "val": { + "rmse": 0.23338687341364966, + "mae": 0.12646035850048065, + "r2": 0.6361653208732605, + "score": -0.23338687341364966 + }, + "test": { + "rmse": 0.23796418466778702, + "mae": 0.1546720266342163, + "r2": 0.613042950630188, + "score": -0.23796418466778702 + } + }, + "time": "0:02:52.457107", + "diversity": { + "train": { + "mean_centered_corr": -0.01820620521903038, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04663686826825142 + }, + "val": { + "mean_centered_corr": -0.01946132257580757, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06672783941030502 + }, + "test": { + "mean_centered_corr": -0.017525583505630493, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09117421507835388 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 18, + 12, + 16, + 4, + 25, + 1, + 8, + 21, + 17, + 4, + 25, + 16, + 12 + ], + "head_selection_time": "0:00:00.219434" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6d426a969 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2378248549747135, + "mae": 0.12229524552822113, + "r2": 0.6386497020721436, + "score": -0.2378248549747135 + }, + "val": { + "rmse": 0.23686825686043878, + "mae": 0.12953220307826996, + "r2": 0.6252298355102539, + "score": -0.23686825686043878 + }, + "test": { + "rmse": 0.24350702848855615, + "mae": 0.15873141586780548, + "r2": 0.5948063731193542, + "score": -0.24350702848855615 + } + }, + "time": "0:02:41.028872", + "diversity": { + "train": { + "mean_centered_corr": -0.018550843000411987, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045891255140304565 + }, + "val": { + "mean_centered_corr": -0.019669199362397194, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06302221864461899 + }, + "test": { + "mean_centered_corr": -0.016400013118982315, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0854080319404602 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015299" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..f2a902eb5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..63e961493 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23087702165729493, + "mae": 0.12027005851268768, + "r2": 0.659454345703125, + "score": -0.23087702165729493 + }, + "val": { + "rmse": 0.2353183042208091, + "mae": 0.12467478215694427, + "r2": 0.6301184296607971, + "score": -0.2353183042208091 + }, + "test": { + "rmse": 0.2326811298212561, + "mae": 0.14326241612434387, + "r2": 0.6300339102745056, + "score": -0.2326811298212561 + } + }, + "time": "0:02:41.028872", + "diversity": { + "train": { + "mean_centered_corr": -0.018550843000411987, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045891255140304565 + }, + "val": { + "mean_centered_corr": -0.019669199362397194, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06302221864461899 + }, + "test": { + "mean_centered_corr": -0.016400013118982315, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0854080319404602 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2376110b7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,220 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.231488372724441, + "mae": 0.12102989852428436, + "r2": 0.657648503780365, + "score": -0.231488372724441 + }, + "val": { + "rmse": 0.23313960483456422, + "mae": 0.12591955065727234, + "r2": 0.636935830116272, + "score": -0.23313960483456422 + }, + "test": { + "rmse": 0.23741637847101169, + "mae": 0.1548576056957245, + "r2": 0.6148224472999573, + "score": -0.23741637847101169 + } + }, + "time": "0:02:41.028872", + "diversity": { + "train": { + "mean_centered_corr": -0.018550843000411987, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045891255140304565 + }, + "val": { + "mean_centered_corr": -0.019669199362397194, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06302221864461899 + }, + "test": { + "mean_centered_corr": -0.016400013118982315, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0854080319404602 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 19, + 28, + 18, + 16, + 25, + 21 + ], + "head_selection_time": "0:00:00.130266" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e7fec36f1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2349271548648021, + "mae": 0.13177132606506348, + "r2": 0.6474015712738037, + "score": -0.2349271548648021 + }, + "val": { + "rmse": 0.23779476251258524, + "mae": 0.12809278070926666, + "r2": 0.6222922801971436, + "score": -0.23779476251258524 + }, + "test": { + "rmse": 0.2405214145563063, + "mae": 0.14176644384860992, + "r2": 0.6046815514564514, + "score": -0.2405214145563063 + } + }, + "time": "0:02:41.928395", + "diversity": { + "train": { + "mean_centered_corr": -0.018742891028523445, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04514782875776291 + }, + "val": { + "mean_centered_corr": -0.018679512664675713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05872314050793648 + }, + "test": { + "mean_centered_corr": -0.01721641607582569, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0779365748167038 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015559" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..d8465a88b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..5e0b1abfa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2304459980871274, + "mae": 0.11964496970176697, + "r2": 0.6607247591018677, + "score": -0.2304459980871274 + }, + "val": { + "rmse": 0.23492211221645545, + "mae": 0.1262272447347641, + "r2": 0.6313629150390625, + "score": -0.23492211221645545 + }, + "test": { + "rmse": 0.23523946918101388, + "mae": 0.148455411195755, + "r2": 0.6218535900115967, + "score": -0.23523946918101388 + } + }, + "time": "0:02:41.928395", + "diversity": { + "train": { + "mean_centered_corr": -0.018742891028523445, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04514782875776291 + }, + "val": { + "mean_centered_corr": -0.018679512664675713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05872314050793648 + }, + "test": { + "mean_centered_corr": -0.01721641607582569, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0779365748167038 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..224b2e6db --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2307565556940205, + "mae": 0.12035396695137024, + "r2": 0.6598096489906311, + "score": -0.2307565556940205 + }, + "val": { + "rmse": 0.2333871447652357, + "mae": 0.12601718306541443, + "r2": 0.6361644268035889, + "score": -0.2333871447652357 + }, + "test": { + "rmse": 0.2345517842823023, + "mae": 0.14844825863838196, + "r2": 0.6240612864494324, + "score": -0.2345517842823023 + } + }, + "time": "0:02:41.928395", + "diversity": { + "train": { + "mean_centered_corr": -0.018742891028523445, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04514782875776291 + }, + "val": { + "mean_centered_corr": -0.018679512664675713, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05872314050793648 + }, + "test": { + "mean_centered_corr": -0.01721641607582569, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0779365748167038 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 0, + 4, + 12, + 28, + 18, + 24 + ], + "head_selection_time": "0:00:00.118122" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..19efdf1b2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2304673922223974, + "mae": 0.13124814629554749, + "r2": 0.6606616973876953, + "score": -0.2304673922223974 + }, + "val": { + "rmse": 0.2389336160662929, + "mae": 0.1372906118631363, + "r2": 0.6186658143997192, + "score": -0.2389336160662929 + }, + "test": { + "rmse": 0.2358168359752649, + "mae": 0.1429797112941742, + "r2": 0.6199951171875, + "score": -0.2358168359752649 + } + }, + "time": "0:02:54.368253", + "diversity": { + "train": { + "mean_centered_corr": -0.018927747383713722, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04583616182208061 + }, + "val": { + "mean_centered_corr": -0.018819812685251236, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06454488635063171 + }, + "test": { + "mean_centered_corr": -0.014754452742636204, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09140031039714813 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:00.015372" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..f0c088e21 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..14df0c853 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22191430166883294, + "mae": 0.11762423813343048, + "r2": 0.6853813529014587, + "score": -0.22191430166883294 + }, + "val": { + "rmse": 0.23496063498765302, + "mae": 0.12751367688179016, + "r2": 0.6312419772148132, + "score": -0.23496063498765302 + }, + "test": { + "rmse": 0.2373818009282627, + "mae": 0.15140388906002045, + "r2": 0.6149346828460693, + "score": -0.2373818009282627 + } + }, + "time": "0:02:54.368253", + "diversity": { + "train": { + "mean_centered_corr": -0.018927747383713722, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04583616182208061 + }, + "val": { + "mean_centered_corr": -0.018819812685251236, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06454488635063171 + }, + "test": { + "mean_centered_corr": -0.014754452742636204, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09140031039714813 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..3c98033c6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,221 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n0p001_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22186501776040204, + "mae": 0.11837654560804367, + "r2": 0.685521125793457, + "score": -0.22186501776040204 + }, + "val": { + "rmse": 0.23389711137908714, + "mae": 0.1278168261051178, + "r2": 0.6345727443695068, + "score": -0.23389711137908714 + }, + "test": { + "rmse": 0.2407960992475978, + "mae": 0.156558558344841, + "r2": 0.6037781238555908, + "score": -0.2407960992475978 + } + }, + "time": "0:02:54.368253", + "diversity": { + "train": { + "mean_centered_corr": -0.018927747383713722, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04583616182208061 + }, + "val": { + "mean_centered_corr": -0.018819812685251236, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06454488635063171 + }, + "test": { + "mean_centered_corr": -0.014754452742636204, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09140031039714813 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 19, + 25, + 12, + 18, + 4, + 13, + 31, + 22 + ], + "head_selection_time": "0:00:00.144189" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..2a1d2c29f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22704868110587859, + "mae": 0.13059158623218536, + "r2": 0.6706544160842896, + "score": -0.22704868110587859 + }, + "val": { + "rmse": 0.23842471915921798, + "mae": 0.13392238318920135, + "r2": 0.6202884912490845, + "score": -0.23842471915921798 + }, + "test": { + "rmse": 0.34030246469362907, + "mae": 0.26470643281936646, + "r2": 0.20864784717559814, + "score": -0.34030246469362907 + } + }, + "time": "0:02:53.902694", + "diversity": { + "train": { + "mean_centered_corr": -0.01857772469520569, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04806574434041977 + }, + "val": { + "mean_centered_corr": -0.018434280529618263, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06849802285432816 + }, + "test": { + "mean_centered_corr": -0.01695123501121998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09515117108821869 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015076" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0.toml new file mode 100644 index 000000000..76382e745 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.00025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0/report.json new file mode 100644 index 000000000..f496496c9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22022479049650132, + "mae": 0.11967124044895172, + "r2": 0.6901537179946899, + "score": -0.22022479049650132 + }, + "val": { + "rmse": 0.23507776598207258, + "mae": 0.1284610629081726, + "r2": 0.6308742761611938, + "score": -0.23507776598207258 + }, + "test": { + "rmse": 0.2438242655494171, + "mae": 0.16113010048866272, + "r2": 0.5937499403953552, + "score": -0.2438242655494171 + } + }, + "time": "0:02:53.902694", + "diversity": { + "train": { + "mean_centered_corr": -0.01857772469520569, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04806574434041977 + }, + "val": { + "mean_centered_corr": -0.018434280529618263, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06849802285432816 + }, + "test": { + "mean_centered_corr": -0.01695123501121998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09515117108821869 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..aa6c9a0ac --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,218 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.00025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p00025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22148010747656677, + "mae": 0.12138405442237854, + "r2": 0.6866112947463989, + "score": -0.22148010747656677 + }, + "val": { + "rmse": 0.23367244804564585, + "mae": 0.127582848072052, + "r2": 0.6352744102478027, + "score": -0.23367244804564585 + }, + "test": { + "rmse": 0.25122008031651116, + "mae": 0.17190353572368622, + "r2": 0.5687309503555298, + "score": -0.25122008031651116 + } + }, + "time": "0:02:53.902694", + "diversity": { + "train": { + "mean_centered_corr": -0.01857772469520569, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04806574434041977 + }, + "val": { + "mean_centered_corr": -0.018434280529618263, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06849802285432816 + }, + "test": { + "mean_centered_corr": -0.01695123501121998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09515117108821869 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 19, + 4, + 21, + 29, + 20 + ], + "head_selection_time": "0:00:00.100741" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e11c2f4c4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2383916630314474, + "mae": 0.12477622181177139, + "r2": 0.6369252800941467, + "score": -0.2383916630314474 + }, + "val": { + "rmse": 0.23743311228743408, + "mae": 0.13437791168689728, + "r2": 0.6234402656555176, + "score": -0.23743311228743408 + }, + "test": { + "rmse": 0.24918858157196577, + "mae": 0.16689999401569366, + "r2": 0.5756776332855225, + "score": -0.24918858157196577 + } + }, + "time": "0:02:39.963879", + "diversity": { + "train": { + "mean_centered_corr": -0.018939347937703133, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04571089893579483 + }, + "val": { + "mean_centered_corr": -0.01955636776983738, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06383588165044785 + }, + "test": { + "mean_centered_corr": -0.01509108766913414, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491861075162888 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015353" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0.toml new file mode 100644 index 000000000..91a0b5da7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0005 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0/report.json new file mode 100644 index 000000000..cd9172ab2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23062436459099714, + "mae": 0.12175119668245316, + "r2": 0.6601992845535278, + "score": -0.23062436459099714 + }, + "val": { + "rmse": 0.23477109182279635, + "mae": 0.12639090418815613, + "r2": 0.6318367123603821, + "score": -0.23477109182279635 + }, + "test": { + "rmse": 0.2340687021877646, + "mae": 0.14674966037273407, + "r2": 0.6256082057952881, + "score": -0.2340687021877646 + } + }, + "time": "0:02:39.963879", + "diversity": { + "train": { + "mean_centered_corr": -0.018939347937703133, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04571089893579483 + }, + "val": { + "mean_centered_corr": -0.01955636776983738, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06383588165044785 + }, + "test": { + "mean_centered_corr": -0.01509108766913414, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491861075162888 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9497066b8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,224 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0005, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p0005-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23107210482218127, + "mae": 0.12221947312355042, + "r2": 0.6588786244392395, + "score": -0.23107210482218127 + }, + "val": { + "rmse": 0.23308539046397073, + "mae": 0.12705643475055695, + "r2": 0.6371046900749207, + "score": -0.23308539046397073 + }, + "test": { + "rmse": 0.23474214725238218, + "mae": 0.15055741369724274, + "r2": 0.623450756072998, + "score": -0.23474214725238218 + } + }, + "time": "0:02:39.963879", + "diversity": { + "train": { + "mean_centered_corr": -0.018939347937703133, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04571089893579483 + }, + "val": { + "mean_centered_corr": -0.01955636776983738, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06383588165044785 + }, + "test": { + "mean_centered_corr": -0.01509108766913414, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08491861075162888 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 18, + 19, + 21, + 12, + 20, + 18, + 16, + 2, + 0, + 10, + 18 + ], + "head_selection_time": "0:00:00.191793" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..37f40c1de --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2255741875961414, + "mae": 0.11985473334789276, + "r2": 0.6749181747436523, + "score": -0.2255741875961414 + }, + "val": { + "rmse": 0.23707897142181117, + "mae": 0.13087449967861176, + "r2": 0.6245627999305725, + "score": -0.23707897142181117 + }, + "test": { + "rmse": 0.23966200804896912, + "mae": 0.15106384456157684, + "r2": 0.6075015068054199, + "score": -0.23966200804896912 + } + }, + "time": "0:02:54.345854", + "diversity": { + "train": { + "mean_centered_corr": -0.018778949975967407, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045425642281770706 + }, + "val": { + "mean_centered_corr": -0.019775288179516792, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06173905357718468 + }, + "test": { + "mean_centered_corr": -0.017589790746569633, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0850314125418663 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 12 + ], + "head_selection_time": "0:00:00.015441" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0.toml new file mode 100644 index 000000000..22cb9039a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.001 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0/report.json new file mode 100644 index 000000000..95dc29351 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22242528246194596, + "mae": 0.11885363608598709, + "r2": 0.6839307546615601, + "score": -0.22242528246194596 + }, + "val": { + "rmse": 0.23488388446999084, + "mae": 0.1276719719171524, + "r2": 0.6314828395843506, + "score": -0.23488388446999084 + }, + "test": { + "rmse": 0.23845471649958624, + "mae": 0.1534513682126999, + "r2": 0.611445963382721, + "score": -0.23845471649958624 + } + }, + "time": "0:02:54.345854", + "diversity": { + "train": { + "mean_centered_corr": -0.018778949975967407, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045425642281770706 + }, + "val": { + "mean_centered_corr": -0.019775288179516792, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06173905357718468 + }, + "test": { + "mean_centered_corr": -0.017589790746569633, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0850314125418663 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ca69ed85d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,217 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.001, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p001-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22164964682417307, + "mae": 0.11660284548997879, + "r2": 0.6861313581466675, + "score": -0.22164964682417307 + }, + "val": { + "rmse": 0.23314020403866242, + "mae": 0.126881942152977, + "r2": 0.6369339823722839, + "score": -0.23314020403866242 + }, + "test": { + "rmse": 0.23450263045082137, + "mae": 0.14819909632205963, + "r2": 0.6242188215255737, + "score": -0.23450263045082137 + } + }, + "time": "0:02:54.345854", + "diversity": { + "train": { + "mean_centered_corr": -0.018778949975967407, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045425642281770706 + }, + "val": { + "mean_centered_corr": -0.019775288179516792, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06173905357718468 + }, + "test": { + "mean_centered_corr": -0.017589790746569633, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0850314125418663 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 12, + 18, + 19, + 1, + 4 + ], + "head_selection_time": "0:00:00.088091" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-best-head-evaluation/0/report.json new file mode 100644 index 000000000..fd4e6f1b5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-best-head-evaluation/0/report.json @@ -0,0 +1,213 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2313265193817783, + "mae": 0.1316187083721161, + "r2": 0.6581270694732666, + "score": -0.2313265193817783 + }, + "val": { + "rmse": 0.2380240960109698, + "mae": 0.13445262610912323, + "r2": 0.6215634346008301, + "score": -0.2380240960109698 + }, + "test": { + "rmse": 0.23356247568168645, + "mae": 0.14156240224838257, + "r2": 0.6272258758544922, + "score": -0.23356247568168645 + } + }, + "time": "0:02:53.865255", + "diversity": { + "train": { + "mean_centered_corr": -0.0185103602707386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0452117882668972 + }, + "val": { + "mean_centered_corr": -0.018912050873041153, + "mean_pairwise_disagreement": NaN, + "member_std": 0.066463403403759 + }, + "test": { + "mean_centered_corr": -0.01741606369614601, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09375505894422531 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16 + ], + "head_selection_time": "0:00:00.015257" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0.toml new file mode 100644 index 000000000..79a2d960c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0.toml @@ -0,0 +1,51 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 +use_esam = true +esam_rho = 0.0025 +esam_eps = 1e-12 +esam_adapter_only = true +esam_memberwise = true +esam_warmup_epochs = 0 +esam_start_epoch = 0 +esam_end_epoch = -1 +esam_log_diagnostics = false +esam_diagnostics_every = 100 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[bins] +n_bins = 84 diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0/report.json new file mode 100644 index 000000000..7b3b8425c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/0/report.json @@ -0,0 +1,209 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22278805228101048, + "mae": 0.11789201945066452, + "r2": 0.6828989386558533, + "score": -0.22278805228101048 + }, + "val": { + "rmse": 0.23480782280356366, + "mae": 0.12755723297595978, + "r2": 0.6317214965820312, + "score": -0.23480782280356366 + }, + "test": { + "rmse": 0.23834319169461882, + "mae": 0.15305806696414948, + "r2": 0.6118093729019165, + "score": -0.23834319169461882 + } + }, + "time": "0:02:53.865255", + "diversity": { + "train": { + "mean_centered_corr": -0.0185103602707386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0452117882668972 + }, + "val": { + "mean_centered_corr": -0.018912050873041153, + "mean_pairwise_disagreement": NaN, + "member_std": 0.066463403403759 + }, + "test": { + "mean_centered_corr": -0.01741606369614601, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09375505894422531 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..df15ecfdf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-greedy-heads-evaluation/0/report.json @@ -0,0 +1,219 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + } + }, + "bins": { + "n_bins": 84 + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": false, + "config": {}, + "mask_stats": { + "keep_rate_actual": 1.0 + } + }, + "cf_fisd": { + "lambda": 0.0 + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": true, + "esam_rho": 0.0025, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": false, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": "/workspace/DL_PROJECT/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation/_esam_diagnostics/sb_comb_rla_esam_r4_n1em05_rho0p0025-evaluation__0.jsonl", + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2236405333806169, + "mae": 0.11948943138122559, + "r2": 0.6804676055908203, + "score": -0.2236405333806169 + }, + "val": { + "rmse": 0.23277917213756988, + "mae": 0.12665784358978271, + "r2": 0.6380575895309448, + "score": -0.23277917213756988 + }, + "test": { + "rmse": 0.2432190859636802, + "mae": 0.16015446186065674, + "r2": 0.5957640409469604, + "score": -0.2432190859636802 + } + }, + "time": "0:02:53.865255", + "diversity": { + "train": { + "mean_centered_corr": -0.0185103602707386, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0452117882668972 + }, + "val": { + "mean_centered_corr": -0.018912050873041153, + "mean_pairwise_disagreement": NaN, + "member_std": 0.066463403403759 + }, + "test": { + "mean_centered_corr": -0.01741606369614601, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09375505894422531 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 16, + 21, + 4, + 22, + 19, + 25, + 7 + ], + "head_selection_time": "0:00:00.116565" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..883d73820 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2314480246211949, + "mae": 0.12086600810289383, + "r2": 0.6577678322792053, + "score": -0.2314480246211949 + }, + "val": { + "rmse": 0.23862914188251422, + "mae": 0.13088369369506836, + "r2": 0.6196370124816895, + "score": -0.23862914188251422 + }, + "test": { + "rmse": 0.23553986357063933, + "mae": 0.14342397451400757, + "r2": 0.6208872199058533, + "score": -0.23553986357063933 + } + }, + "time": "0:01:16.306157", + "diversity": { + "train": { + "mean_centered_corr": -0.019465479999780655, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04585251957178116 + }, + "val": { + "mean_centered_corr": -0.01962343417108059, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06466832011938095 + }, + "test": { + "mean_centered_corr": -0.016597073525190353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08519241958856583 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015051" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0.toml new file mode 100644 index 000000000..c0216f239 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..188a924d4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22484523994052877, + "mae": 0.1174779012799263, + "r2": 0.6770157814025879, + "score": -0.22484523994052877 + }, + "val": { + "rmse": 0.2347666170730829, + "mae": 0.1269145905971527, + "r2": 0.6318507194519043, + "score": -0.2347666170730829 + }, + "test": { + "rmse": 0.23399684917333807, + "mae": 0.14828696846961975, + "r2": 0.625838041305542, + "score": -0.23399684917333807 + } + }, + "time": "0:01:16.306157", + "diversity": { + "train": { + "mean_centered_corr": -0.019465479999780655, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04585251957178116 + }, + "val": { + "mean_centered_corr": -0.01962343417108059, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06466832011938095 + }, + "test": { + "mean_centered_corr": -0.016597073525190353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08519241958856583 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0b3bb4881 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,691 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22569541421502587, + "mae": 0.11816450208425522, + "r2": 0.6745686531066895, + "score": -0.22569541421502587 + }, + "val": { + "rmse": 0.23305490182627153, + "mae": 0.1270279735326767, + "r2": 0.6371996402740479, + "score": -0.23305490182627153 + }, + "test": { + "rmse": 0.23732565171059752, + "mae": 0.15259529650211334, + "r2": 0.6151168346405029, + "score": -0.23732565171059752 + } + }, + "time": "0:01:16.306157", + "diversity": { + "train": { + "mean_centered_corr": -0.019465479999780655, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04585251957178116 + }, + "val": { + "mean_centered_corr": -0.01962343417108059, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06466832011938095 + }, + "test": { + "mean_centered_corr": -0.016597073525190353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08519241958856583 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 25, + 21, + 29 + ], + "head_selection_time": "0:00:00.072510" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..675c9212e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2241888421744036, + "mae": 0.1289592683315277, + "r2": 0.678898811340332, + "score": -0.2241888421744036 + }, + "val": { + "rmse": 0.23918568828057105, + "mae": 0.14043618738651276, + "r2": 0.6178607940673828, + "score": -0.23918568828057105 + }, + "test": { + "rmse": 0.25293341379845147, + "mae": 0.17517992854118347, + "r2": 0.5628283023834229, + "score": -0.25293341379845147 + } + }, + "time": "0:01:18.920555", + "diversity": { + "train": { + "mean_centered_corr": -0.017244398593902588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045789685100317 + }, + "val": { + "mean_centered_corr": -0.019243216142058372, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06452018767595291 + }, + "test": { + "mean_centered_corr": -0.017609506845474243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08646101504564285 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015213" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0.toml new file mode 100644 index 000000000..5c1e3b792 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..8707168a8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.220642866499973, + "mae": 0.11938578635454178, + "r2": 0.6889761686325073, + "score": -0.220642866499973 + }, + "val": { + "rmse": 0.2351252944190024, + "mae": 0.12928679585456848, + "r2": 0.6307249069213867, + "score": -0.2351252944190024 + }, + "test": { + "rmse": 0.23697534320527558, + "mae": 0.15434296429157257, + "r2": 0.6162521839141846, + "score": -0.23697534320527558 + } + }, + "time": "0:01:18.920555", + "diversity": { + "train": { + "mean_centered_corr": -0.017244398593902588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045789685100317 + }, + "val": { + "mean_centered_corr": -0.019243216142058372, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06452018767595291 + }, + "test": { + "mean_centered_corr": -0.017609506845474243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08646101504564285 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..34be5ef29 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,693 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2206414904658828, + "mae": 0.12224060297012329, + "r2": 0.6889800429344177, + "score": -0.2206414904658828 + }, + "val": { + "rmse": 0.2331196304833006, + "mae": 0.12830887734889984, + "r2": 0.6369980573654175, + "score": -0.2331196304833006 + }, + "test": { + "rmse": 0.23280496841127477, + "mae": 0.14530567824840546, + "r2": 0.6296399831771851, + "score": -0.23280496841127477 + } + }, + "time": "0:01:18.920555", + "diversity": { + "train": { + "mean_centered_corr": -0.017244398593902588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045789685100317 + }, + "val": { + "mean_centered_corr": -0.019243216142058372, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06452018767595291 + }, + "test": { + "mean_centered_corr": -0.017609506845474243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08646101504564285 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 8, + 20, + 31, + 19, + 0 + ], + "head_selection_time": "0:00:00.102311" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5be2251b0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2425666348719968, + "mae": 0.12444926798343658, + "r2": 0.6240968108177185, + "score": -0.2425666348719968 + }, + "val": { + "rmse": 0.23756948853650636, + "mae": 0.13087332248687744, + "r2": 0.6230076551437378, + "score": -0.23756948853650636 + }, + "test": { + "rmse": 0.24224213014777235, + "mae": 0.15561068058013916, + "r2": 0.5990049839019775, + "score": -0.24224213014777235 + } + }, + "time": "0:01:10.287046", + "diversity": { + "train": { + "mean_centered_corr": -0.019360000267624855, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683505743741989 + }, + "val": { + "mean_centered_corr": -0.019574038684368134, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06655079871416092 + }, + "test": { + "mean_centered_corr": -0.01754024624824524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08368789404630661 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015481" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0.toml new file mode 100644 index 000000000..c0b3f5a8d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..ad67df3e9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23343200123041558, + "mae": 0.12027841061353683, + "r2": 0.6518754363059998, + "score": -0.23343200123041558 + }, + "val": { + "rmse": 0.23512438339576403, + "mae": 0.12516294419765472, + "r2": 0.6307278275489807, + "score": -0.23512438339576403 + }, + "test": { + "rmse": 0.2316782780837399, + "mae": 0.14233005046844482, + "r2": 0.633216142654419, + "score": -0.2316782780837399 + } + }, + "time": "0:01:10.287046", + "diversity": { + "train": { + "mean_centered_corr": -0.019360000267624855, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683505743741989 + }, + "val": { + "mean_centered_corr": -0.019574038684368134, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06655079871416092 + }, + "test": { + "mean_centered_corr": -0.01754024624824524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08368789404630661 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..41e4d69c6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,693 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23458808907797565, + "mae": 0.12325569242238998, + "r2": 0.648418664932251, + "score": -0.23458808907797565 + }, + "val": { + "rmse": 0.23291770554949479, + "mae": 0.1261579543352127, + "r2": 0.6376266479492188, + "score": -0.23291770554949479 + }, + "test": { + "rmse": 0.23458472246540202, + "mae": 0.14723080396652222, + "r2": 0.6239557266235352, + "score": -0.23458472246540202 + } + }, + "time": "0:01:10.287046", + "diversity": { + "train": { + "mean_centered_corr": -0.019360000267624855, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683505743741989 + }, + "val": { + "mean_centered_corr": -0.019574038684368134, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06655079871416092 + }, + "test": { + "mean_centered_corr": -0.01754024624824524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08368789404630661 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 25, + 21, + 6, + 19, + 0 + ], + "head_selection_time": "0:00:00.103333" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..36b5eaaf3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24039030054595545, + "mae": 0.1237230896949768, + "r2": 0.6308118104934692, + "score": -0.24039030054595545 + }, + "val": { + "rmse": 0.23560357747729638, + "mae": 0.13029368221759796, + "r2": 0.6292210817337036, + "score": -0.23560357747729638 + }, + "test": { + "rmse": 0.2389191702722735, + "mae": 0.15265432000160217, + "r2": 0.6099308729171753, + "score": -0.2389191702722735 + } + }, + "time": "0:01:08.862892", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015418" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml new file mode 100644 index 000000000..3417f3906 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..19c46df9c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23278188472143527, + "mae": 0.12037675827741623, + "r2": 0.6538118124008179, + "score": -0.23278188472143527 + }, + "val": { + "rmse": 0.235050071622046, + "mae": 0.12535566091537476, + "r2": 0.6309611797332764, + "score": -0.235050071622046 + }, + "test": { + "rmse": 0.23184680318718037, + "mae": 0.14313000440597534, + "r2": 0.6326823234558105, + "score": -0.23184680318718037 + } + }, + "time": "0:01:08.862892", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ec8575dd0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,692 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23573796221874224, + "mae": 0.12117554992437363, + "r2": 0.6449635028839111, + "score": -0.23573796221874224 + }, + "val": { + "rmse": 0.2324412121462238, + "mae": 0.12522703409194946, + "r2": 0.6391078233718872, + "score": -0.2324412121462238 + }, + "test": { + "rmse": 0.23820740696857753, + "mae": 0.1556985229253769, + "r2": 0.6122515201568604, + "score": -0.23820740696857753 + } + }, + "time": "0:01:08.862892", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 1, + 25, + 21 + ], + "head_selection_time": "0:00:00.088435" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..88521f57d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24104706607058576, + "mae": 0.1307125687599182, + "r2": 0.6287918090820312, + "score": -0.24104706607058576 + }, + "val": { + "rmse": 0.2372450969584346, + "mae": 0.1306145042181015, + "r2": 0.624036431312561, + "score": -0.2372450969584346 + }, + "test": { + "rmse": 0.23726047684878263, + "mae": 0.14689351618289948, + "r2": 0.6153281927108765, + "score": -0.23726047684878263 + } + }, + "time": "0:01:10.655002", + "diversity": { + "train": { + "mean_centered_corr": -0.019136181101202965, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04613374546170235 + }, + "val": { + "mean_centered_corr": -0.01984967105090618, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134215369820595 + }, + "test": { + "mean_centered_corr": -0.0187427569180727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017097413539886 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015354" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0.toml new file mode 100644 index 000000000..5b20437d1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..78f62d5da --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23274665867971067, + "mae": 0.1198442280292511, + "r2": 0.653916597366333, + "score": -0.23274665867971067 + }, + "val": { + "rmse": 0.23505585640746507, + "mae": 0.12548410892486572, + "r2": 0.6309430599212646, + "score": -0.23505585640746507 + }, + "test": { + "rmse": 0.23276235981389973, + "mae": 0.14522086083889008, + "r2": 0.6297755241394043, + "score": -0.23276235981389973 + } + }, + "time": "0:01:10.655002", + "diversity": { + "train": { + "mean_centered_corr": -0.019136181101202965, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04613374546170235 + }, + "val": { + "mean_centered_corr": -0.01984967105090618, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134215369820595 + }, + "test": { + "mean_centered_corr": -0.0187427569180727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017097413539886 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8eed36788 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,696 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23595696457409665, + "mae": 0.12094593048095703, + "r2": 0.644303560256958, + "score": -0.23595696457409665 + }, + "val": { + "rmse": 0.2328726380397863, + "mae": 0.12572719156742096, + "r2": 0.6377668380737305, + "score": -0.2328726380397863 + }, + "test": { + "rmse": 0.23808236498886684, + "mae": 0.15540435910224915, + "r2": 0.6126585006713867, + "score": -0.23808236498886684 + } + }, + "time": "0:01:10.655002", + "diversity": { + "train": { + "mean_centered_corr": -0.019136181101202965, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04613374546170235 + }, + "val": { + "mean_centered_corr": -0.01984967105090618, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134215369820595 + }, + "test": { + "mean_centered_corr": -0.0187427569180727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017097413539886 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 4, + 16, + 1, + 10, + 25, + 21, + 16, + 4 + ], + "head_selection_time": "0:00:00.146234" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7ceb5b4b6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24078892072406902, + "mae": 0.12140180170536041, + "r2": 0.6295864582061768, + "score": -0.24078892072406902 + }, + "val": { + "rmse": 0.23656618722639788, + "mae": 0.1301935315132141, + "r2": 0.6261851191520691, + "score": -0.23656618722639788 + }, + "test": { + "rmse": 0.2443526301182123, + "mae": 0.15885169804096222, + "r2": 0.5919873714447021, + "score": -0.2443526301182123 + } + }, + "time": "0:01:09.764472", + "diversity": { + "train": { + "mean_centered_corr": -0.019902383908629417, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04721210524439812 + }, + "val": { + "mean_centered_corr": -0.019949622452259064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06361114978790283 + }, + "test": { + "mean_centered_corr": -0.019116314128041267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07747960835695267 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015310" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0.toml new file mode 100644 index 000000000..3040481ce --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..a4a16853f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23225320578180678, + "mae": 0.12026127427816391, + "r2": 0.655382513999939, + "score": -0.23225320578180678 + }, + "val": { + "rmse": 0.23504784483792887, + "mae": 0.12635678052902222, + "r2": 0.6309682130813599, + "score": -0.23504784483792887 + }, + "test": { + "rmse": 0.23465037063607563, + "mae": 0.14887936413288116, + "r2": 0.6237452030181885, + "score": -0.23465037063607563 + } + }, + "time": "0:01:09.764472", + "diversity": { + "train": { + "mean_centered_corr": -0.019902383908629417, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04721210524439812 + }, + "val": { + "mean_centered_corr": -0.019949622452259064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06361114978790283 + }, + "test": { + "mean_centered_corr": -0.019116314128041267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07747960835695267 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5829f6ed4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,695 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2344874747922904, + "mae": 0.1211647242307663, + "r2": 0.6487202048301697, + "score": -0.2344874747922904 + }, + "val": { + "rmse": 0.23334387616340208, + "mae": 0.12681756913661957, + "r2": 0.6362993717193604, + "score": -0.23334387616340208 + }, + "test": { + "rmse": 0.2380340654227449, + "mae": 0.15519878268241882, + "r2": 0.6128156185150146, + "score": -0.2380340654227449 + } + }, + "time": "0:01:09.764472", + "diversity": { + "train": { + "mean_centered_corr": -0.019902383908629417, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04721210524439812 + }, + "val": { + "mean_centered_corr": -0.019949622452259064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06361114978790283 + }, + "test": { + "mean_centered_corr": -0.019116314128041267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07747960835695267 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 29, + 19, + 1, + 10, + 25, + 21, + 16 + ], + "head_selection_time": "0:00:00.131498" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f83824fc1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24066084712400115, + "mae": 0.1302179992198944, + "r2": 0.6299803256988525, + "score": -0.24066084712400115 + }, + "val": { + "rmse": 0.23897181954001903, + "mae": 0.13393160700798035, + "r2": 0.6185438632965088, + "score": -0.23897181954001903 + }, + "test": { + "rmse": 0.23496733361066027, + "mae": 0.1394679695367813, + "r2": 0.6227279901504517, + "score": -0.23496733361066027 + } + }, + "time": "0:01:11.476918", + "diversity": { + "train": { + "mean_centered_corr": -0.01913526840507984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782585799694061 + }, + "val": { + "mean_centered_corr": -0.01699827052652836, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06600741297006607 + }, + "test": { + "mean_centered_corr": -0.020212531089782715, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08567789196968079 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.015265" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0.toml new file mode 100644 index 000000000..425700ac8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..f76ddab00 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2318619145508088, + "mae": 0.12150537222623825, + "r2": 0.6565427184104919, + "score": -0.2318619145508088 + }, + "val": { + "rmse": 0.2357328737026148, + "mae": 0.12986169755458832, + "r2": 0.6288140416145325, + "score": -0.2357328737026148 + }, + "test": { + "rmse": 0.2351772485906963, + "mae": 0.14941059052944183, + "r2": 0.6220536231994629, + "score": -0.2351772485906963 + } + }, + "time": "0:01:11.476918", + "diversity": { + "train": { + "mean_centered_corr": -0.01913526840507984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782585799694061 + }, + "val": { + "mean_centered_corr": -0.01699827052652836, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06600741297006607 + }, + "test": { + "mean_centered_corr": -0.020212531089782715, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08567789196968079 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..39e1ab40b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23260143300346486, + "mae": 0.12271209806203842, + "r2": 0.6543483734130859, + "score": -0.23260143300346486 + }, + "val": { + "rmse": 0.2341784288692296, + "mae": 0.1288924217224121, + "r2": 0.6336931586265564, + "score": -0.2341784288692296 + }, + "test": { + "rmse": 0.23454983071534766, + "mae": 0.14984872937202454, + "r2": 0.6240675449371338, + "score": -0.23454983071534766 + } + }, + "time": "0:01:11.476918", + "diversity": { + "train": { + "mean_centered_corr": -0.01913526840507984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782585799694061 + }, + "val": { + "mean_centered_corr": -0.01699827052652836, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06600741297006607 + }, + "test": { + "mean_centered_corr": -0.020212531089782715, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08567789196968079 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8, + 21, + 27, + 19, + 3, + 25, + 4, + 12, + 0, + 18, + 3, + 24 + ], + "head_selection_time": "0:00:00.187545" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0a3243787 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.22160140516469537, + "mae": 0.12272227555513382, + "r2": 0.6862679719924927, + "score": -0.22160140516469537 + }, + "val": { + "rmse": 0.24058462987099405, + "mae": 0.13246983289718628, + "r2": 0.613377571105957, + "score": -0.24058462987099405 + }, + "test": { + "rmse": 0.2859647081281448, + "mae": 0.2032262533903122, + "r2": 0.44118958711624146, + "score": -0.2859647081281448 + } + }, + "time": "0:01:31.771928", + "diversity": { + "train": { + "mean_centered_corr": -0.0177434291690588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05118265748023987 + }, + "val": { + "mean_centered_corr": -0.01485136803239584, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0833396464586258 + }, + "test": { + "mean_centered_corr": -0.015522093512117863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11537164449691772 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 10 + ], + "head_selection_time": "0:00:00.015141" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0.toml new file mode 100644 index 000000000..a62285992 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..a866deda9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.2047007403568582, + "mae": 0.11088933050632477, + "r2": 0.7322973012924194, + "score": -0.2047007403568582 + }, + "val": { + "rmse": 0.2351519581095747, + "mae": 0.12822481989860535, + "r2": 0.6306412220001221, + "score": -0.2351519581095747 + }, + "test": { + "rmse": 0.2408751648791966, + "mae": 0.15764909982681274, + "r2": 0.6035178899765015, + "score": -0.2408751648791966 + } + }, + "time": "0:01:31.771928", + "diversity": { + "train": { + "mean_centered_corr": -0.0177434291690588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05118265748023987 + }, + "val": { + "mean_centered_corr": -0.01485136803239584, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0833396464586258 + }, + "test": { + "mean_centered_corr": -0.015522093512117863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11537164449691772 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ece2cef9e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,705 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20677708372923248, + "mae": 0.11196121573448181, + "r2": 0.7268389463424683, + "score": -0.20677708372923248 + }, + "val": { + "rmse": 0.23316436266516308, + "mae": 0.12652936577796936, + "r2": 0.6368587017059326, + "score": -0.23316436266516308 + }, + "test": { + "rmse": 0.23171057167720172, + "mae": 0.14218251407146454, + "r2": 0.6331138610839844, + "score": -0.23171057167720172 + } + }, + "time": "0:01:31.771928", + "diversity": { + "train": { + "mean_centered_corr": -0.0177434291690588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05118265748023987 + }, + "val": { + "mean_centered_corr": -0.01485136803239584, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0833396464586258 + }, + "test": { + "mean_centered_corr": -0.015522093512117863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11537164449691772 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 10, + 27, + 7, + 1, + 23, + 16, + 29, + 5, + 17, + 7, + 25, + 29, + 20, + 27, + 1, + 21, + 7, + 22 + ], + "head_selection_time": "0:00:00.281144" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..504a3c981 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2438513529798656, + "mae": 0.12642574310302734, + "r2": 0.6201044321060181, + "score": -0.2438513529798656 + }, + "val": { + "rmse": 0.23781902006036262, + "mae": 0.13461825251579285, + "r2": 0.6222152709960938, + "score": -0.23781902006036262 + }, + "test": { + "rmse": 0.2470883298795412, + "mae": 0.16668537259101868, + "r2": 0.5828002095222473, + "score": -0.2470883298795412 + } + }, + "time": "0:01:11.254152", + "diversity": { + "train": { + "mean_centered_corr": -0.01934780180454254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04824277386069298 + }, + "val": { + "mean_centered_corr": -0.018993837758898735, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06548074632883072 + }, + "test": { + "mean_centered_corr": -0.019981293007731438, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0883246660232544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015371" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0.toml new file mode 100644 index 000000000..9a8e1d1e5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..599624835 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23279507911071862, + "mae": 0.12109043449163437, + "r2": 0.6537725925445557, + "score": -0.23279507911071862 + }, + "val": { + "rmse": 0.2353263382433216, + "mae": 0.12844589352607727, + "r2": 0.6300932168960571, + "score": -0.2353263382433216 + }, + "test": { + "rmse": 0.2351845825591179, + "mae": 0.1493937373161316, + "r2": 0.6220300197601318, + "score": -0.2351845825591179 + } + }, + "time": "0:01:11.254152", + "diversity": { + "train": { + "mean_centered_corr": -0.01934780180454254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04824277386069298 + }, + "val": { + "mean_centered_corr": -0.018993837758898735, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06548074632883072 + }, + "test": { + "mean_centered_corr": -0.019981293007731438, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0883246660232544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..e79972df5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23261467766548827, + "mae": 0.12115577608346939, + "r2": 0.6543089747428894, + "score": -0.23261467766548827 + }, + "val": { + "rmse": 0.2335463179080623, + "mae": 0.127370223402977, + "r2": 0.6356680393218994, + "score": -0.2335463179080623 + }, + "test": { + "rmse": 0.23522276143697368, + "mae": 0.15016508102416992, + "r2": 0.6219072937965393, + "score": -0.23522276143697368 + } + }, + "time": "0:01:11.254152", + "diversity": { + "train": { + "mean_centered_corr": -0.01934780180454254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04824277386069298 + }, + "val": { + "mean_centered_corr": -0.018993837758898735, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06548074632883072 + }, + "test": { + "mean_centered_corr": -0.019981293007731438, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0883246660232544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 19, + 1, + 20, + 25, + 12, + 21, + 25, + 3, + 28, + 19 + ], + "head_selection_time": "0:00:00.175756" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9723c36d4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2314480246211949, + "mae": 0.12086600810289383, + "r2": 0.6577678322792053, + "score": -0.2314480246211949 + }, + "val": { + "rmse": 0.23862914188251422, + "mae": 0.13088369369506836, + "r2": 0.6196370124816895, + "score": -0.23862914188251422 + }, + "test": { + "rmse": 0.23553986357063933, + "mae": 0.14342397451400757, + "r2": 0.6208872199058533, + "score": -0.23553986357063933 + } + }, + "time": "0:01:15.113893", + "diversity": { + "train": { + "mean_centered_corr": -0.019465479999780655, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04585251957178116 + }, + "val": { + "mean_centered_corr": -0.01962343417108059, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06466832011938095 + }, + "test": { + "mean_centered_corr": -0.016597073525190353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08519241958856583 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015247" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0.toml new file mode 100644 index 000000000..48ad2cfe9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..9c1d0f680 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22484523994052877, + "mae": 0.1174779012799263, + "r2": 0.6770157814025879, + "score": -0.22484523994052877 + }, + "val": { + "rmse": 0.2347666170730829, + "mae": 0.1269145905971527, + "r2": 0.6318507194519043, + "score": -0.2347666170730829 + }, + "test": { + "rmse": 0.23399684917333807, + "mae": 0.14828696846961975, + "r2": 0.625838041305542, + "score": -0.23399684917333807 + } + }, + "time": "0:01:15.113893", + "diversity": { + "train": { + "mean_centered_corr": -0.019465479999780655, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04585251957178116 + }, + "val": { + "mean_centered_corr": -0.01962343417108059, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06466832011938095 + }, + "test": { + "mean_centered_corr": -0.016597073525190353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08519241958856583 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5df4cd600 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,691 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22569541421502587, + "mae": 0.11816450208425522, + "r2": 0.6745686531066895, + "score": -0.22569541421502587 + }, + "val": { + "rmse": 0.23305490182627153, + "mae": 0.1270279735326767, + "r2": 0.6371996402740479, + "score": -0.23305490182627153 + }, + "test": { + "rmse": 0.23732565171059752, + "mae": 0.15259529650211334, + "r2": 0.6151168346405029, + "score": -0.23732565171059752 + } + }, + "time": "0:01:15.113893", + "diversity": { + "train": { + "mean_centered_corr": -0.019465479999780655, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04585251957178116 + }, + "val": { + "mean_centered_corr": -0.01962343417108059, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06466832011938095 + }, + "test": { + "mean_centered_corr": -0.016597073525190353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08519241958856583 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 25, + 21, + 29 + ], + "head_selection_time": "0:00:00.073868" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ef0efb8bb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2241888421744036, + "mae": 0.1289592683315277, + "r2": 0.678898811340332, + "score": -0.2241888421744036 + }, + "val": { + "rmse": 0.23918568828057105, + "mae": 0.14043618738651276, + "r2": 0.6178607940673828, + "score": -0.23918568828057105 + }, + "test": { + "rmse": 0.25293341379845147, + "mae": 0.17517992854118347, + "r2": 0.5628283023834229, + "score": -0.25293341379845147 + } + }, + "time": "0:01:20.898855", + "diversity": { + "train": { + "mean_centered_corr": -0.017244398593902588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045789685100317 + }, + "val": { + "mean_centered_corr": -0.019243216142058372, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06452018767595291 + }, + "test": { + "mean_centered_corr": -0.017609506845474243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08646101504564285 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015361" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0.toml new file mode 100644 index 000000000..8db1064c6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..eb10527ef --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.220642866499973, + "mae": 0.11938578635454178, + "r2": 0.6889761686325073, + "score": -0.220642866499973 + }, + "val": { + "rmse": 0.2351252944190024, + "mae": 0.12928679585456848, + "r2": 0.6307249069213867, + "score": -0.2351252944190024 + }, + "test": { + "rmse": 0.23697534320527558, + "mae": 0.15434296429157257, + "r2": 0.6162521839141846, + "score": -0.23697534320527558 + } + }, + "time": "0:01:20.898855", + "diversity": { + "train": { + "mean_centered_corr": -0.017244398593902588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045789685100317 + }, + "val": { + "mean_centered_corr": -0.019243216142058372, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06452018767595291 + }, + "test": { + "mean_centered_corr": -0.017609506845474243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08646101504564285 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..24faab8e3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,693 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2206414904658828, + "mae": 0.12224060297012329, + "r2": 0.6889800429344177, + "score": -0.2206414904658828 + }, + "val": { + "rmse": 0.2331196304833006, + "mae": 0.12830887734889984, + "r2": 0.6369980573654175, + "score": -0.2331196304833006 + }, + "test": { + "rmse": 0.23280496841127477, + "mae": 0.14530567824840546, + "r2": 0.6296399831771851, + "score": -0.23280496841127477 + } + }, + "time": "0:01:20.898855", + "diversity": { + "train": { + "mean_centered_corr": -0.017244398593902588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045789685100317 + }, + "val": { + "mean_centered_corr": -0.019243216142058372, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06452018767595291 + }, + "test": { + "mean_centered_corr": -0.017609506845474243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08646101504564285 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 8, + 20, + 31, + 19, + 0 + ], + "head_selection_time": "0:00:00.101908" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ae99c67f7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2425666348719968, + "mae": 0.12444926798343658, + "r2": 0.6240968108177185, + "score": -0.2425666348719968 + }, + "val": { + "rmse": 0.23756948853650636, + "mae": 0.13087332248687744, + "r2": 0.6230076551437378, + "score": -0.23756948853650636 + }, + "test": { + "rmse": 0.24224213014777235, + "mae": 0.15561068058013916, + "r2": 0.5990049839019775, + "score": -0.24224213014777235 + } + }, + "time": "0:01:11.295558", + "diversity": { + "train": { + "mean_centered_corr": -0.019360000267624855, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683505743741989 + }, + "val": { + "mean_centered_corr": -0.019574038684368134, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06655079871416092 + }, + "test": { + "mean_centered_corr": -0.01754024624824524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08368789404630661 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015347" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0.toml new file mode 100644 index 000000000..2636255e4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..73e8ea7c3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23343200123041558, + "mae": 0.12027841061353683, + "r2": 0.6518754363059998, + "score": -0.23343200123041558 + }, + "val": { + "rmse": 0.23512438339576403, + "mae": 0.12516294419765472, + "r2": 0.6307278275489807, + "score": -0.23512438339576403 + }, + "test": { + "rmse": 0.2316782780837399, + "mae": 0.14233005046844482, + "r2": 0.633216142654419, + "score": -0.2316782780837399 + } + }, + "time": "0:01:11.295558", + "diversity": { + "train": { + "mean_centered_corr": -0.019360000267624855, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683505743741989 + }, + "val": { + "mean_centered_corr": -0.019574038684368134, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06655079871416092 + }, + "test": { + "mean_centered_corr": -0.01754024624824524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08368789404630661 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..63be52b28 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,693 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23458808907797565, + "mae": 0.12325569242238998, + "r2": 0.648418664932251, + "score": -0.23458808907797565 + }, + "val": { + "rmse": 0.23291770554949479, + "mae": 0.1261579543352127, + "r2": 0.6376266479492188, + "score": -0.23291770554949479 + }, + "test": { + "rmse": 0.23458472246540202, + "mae": 0.14723080396652222, + "r2": 0.6239557266235352, + "score": -0.23458472246540202 + } + }, + "time": "0:01:11.295558", + "diversity": { + "train": { + "mean_centered_corr": -0.019360000267624855, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683505743741989 + }, + "val": { + "mean_centered_corr": -0.019574038684368134, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06655079871416092 + }, + "test": { + "mean_centered_corr": -0.01754024624824524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08368789404630661 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 25, + 21, + 6, + 19, + 0 + ], + "head_selection_time": "0:00:00.101499" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3fbae2439 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24039030054595545, + "mae": 0.1237230896949768, + "r2": 0.6308118104934692, + "score": -0.24039030054595545 + }, + "val": { + "rmse": 0.23560357747729638, + "mae": 0.13029368221759796, + "r2": 0.6292210817337036, + "score": -0.23560357747729638 + }, + "test": { + "rmse": 0.2389191702722735, + "mae": 0.15265432000160217, + "r2": 0.6099308729171753, + "score": -0.2389191702722735 + } + }, + "time": "0:01:11.031390", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015409" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0.toml new file mode 100644 index 000000000..6cf1a5f53 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..bf5dfa8ef --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23278188472143527, + "mae": 0.12037675827741623, + "r2": 0.6538118124008179, + "score": -0.23278188472143527 + }, + "val": { + "rmse": 0.235050071622046, + "mae": 0.12535566091537476, + "r2": 0.6309611797332764, + "score": -0.235050071622046 + }, + "test": { + "rmse": 0.23184680318718037, + "mae": 0.14313000440597534, + "r2": 0.6326823234558105, + "score": -0.23184680318718037 + } + }, + "time": "0:01:11.031390", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..134f955c4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,692 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23573796221874224, + "mae": 0.12117554992437363, + "r2": 0.6449635028839111, + "score": -0.23573796221874224 + }, + "val": { + "rmse": 0.2324412121462238, + "mae": 0.12522703409194946, + "r2": 0.6391078233718872, + "score": -0.2324412121462238 + }, + "test": { + "rmse": 0.23820740696857753, + "mae": 0.1556985229253769, + "r2": 0.6122515201568604, + "score": -0.23820740696857753 + } + }, + "time": "0:01:11.031390", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 1, + 25, + 21 + ], + "head_selection_time": "0:00:00.088364" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..43a45a359 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24104706607058576, + "mae": 0.1307125687599182, + "r2": 0.6287918090820312, + "score": -0.24104706607058576 + }, + "val": { + "rmse": 0.2372450969584346, + "mae": 0.1306145042181015, + "r2": 0.624036431312561, + "score": -0.2372450969584346 + }, + "test": { + "rmse": 0.23726047684878263, + "mae": 0.14689351618289948, + "r2": 0.6153281927108765, + "score": -0.23726047684878263 + } + }, + "time": "0:01:12.101991", + "diversity": { + "train": { + "mean_centered_corr": -0.019136181101202965, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04613374546170235 + }, + "val": { + "mean_centered_corr": -0.01984967105090618, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134215369820595 + }, + "test": { + "mean_centered_corr": -0.0187427569180727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017097413539886 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015275" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0.toml new file mode 100644 index 000000000..ed5045f78 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..caca58912 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23274665867971067, + "mae": 0.1198442280292511, + "r2": 0.653916597366333, + "score": -0.23274665867971067 + }, + "val": { + "rmse": 0.23505585640746507, + "mae": 0.12548410892486572, + "r2": 0.6309430599212646, + "score": -0.23505585640746507 + }, + "test": { + "rmse": 0.23276235981389973, + "mae": 0.14522086083889008, + "r2": 0.6297755241394043, + "score": -0.23276235981389973 + } + }, + "time": "0:01:12.101991", + "diversity": { + "train": { + "mean_centered_corr": -0.019136181101202965, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04613374546170235 + }, + "val": { + "mean_centered_corr": -0.01984967105090618, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134215369820595 + }, + "test": { + "mean_centered_corr": -0.0187427569180727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017097413539886 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d62e44a81 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,696 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23595696457409665, + "mae": 0.12094593048095703, + "r2": 0.644303560256958, + "score": -0.23595696457409665 + }, + "val": { + "rmse": 0.2328726380397863, + "mae": 0.12572719156742096, + "r2": 0.6377668380737305, + "score": -0.2328726380397863 + }, + "test": { + "rmse": 0.23808236498886684, + "mae": 0.15540435910224915, + "r2": 0.6126585006713867, + "score": -0.23808236498886684 + } + }, + "time": "0:01:12.101991", + "diversity": { + "train": { + "mean_centered_corr": -0.019136181101202965, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04613374546170235 + }, + "val": { + "mean_centered_corr": -0.01984967105090618, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134215369820595 + }, + "test": { + "mean_centered_corr": -0.0187427569180727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017097413539886 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 4, + 16, + 1, + 10, + 25, + 21, + 16, + 4 + ], + "head_selection_time": "0:00:00.146138" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8e82cc8bc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24078892072406902, + "mae": 0.12140180170536041, + "r2": 0.6295864582061768, + "score": -0.24078892072406902 + }, + "val": { + "rmse": 0.23656618722639788, + "mae": 0.1301935315132141, + "r2": 0.6261851191520691, + "score": -0.23656618722639788 + }, + "test": { + "rmse": 0.2443526301182123, + "mae": 0.15885169804096222, + "r2": 0.5919873714447021, + "score": -0.2443526301182123 + } + }, + "time": "0:01:10.136519", + "diversity": { + "train": { + "mean_centered_corr": -0.019902383908629417, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04721210524439812 + }, + "val": { + "mean_centered_corr": -0.019949622452259064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06361114978790283 + }, + "test": { + "mean_centered_corr": -0.019116314128041267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07747960835695267 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015011" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0.toml new file mode 100644 index 000000000..cc09b9d7d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..aa3dfb801 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23225320578180678, + "mae": 0.12026127427816391, + "r2": 0.655382513999939, + "score": -0.23225320578180678 + }, + "val": { + "rmse": 0.23504784483792887, + "mae": 0.12635678052902222, + "r2": 0.6309682130813599, + "score": -0.23504784483792887 + }, + "test": { + "rmse": 0.23465037063607563, + "mae": 0.14887936413288116, + "r2": 0.6237452030181885, + "score": -0.23465037063607563 + } + }, + "time": "0:01:10.136519", + "diversity": { + "train": { + "mean_centered_corr": -0.019902383908629417, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04721210524439812 + }, + "val": { + "mean_centered_corr": -0.019949622452259064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06361114978790283 + }, + "test": { + "mean_centered_corr": -0.019116314128041267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07747960835695267 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1ddbc24cd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,695 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2344874747922904, + "mae": 0.1211647242307663, + "r2": 0.6487202048301697, + "score": -0.2344874747922904 + }, + "val": { + "rmse": 0.23334387616340208, + "mae": 0.12681756913661957, + "r2": 0.6362993717193604, + "score": -0.23334387616340208 + }, + "test": { + "rmse": 0.2380340654227449, + "mae": 0.15519878268241882, + "r2": 0.6128156185150146, + "score": -0.2380340654227449 + } + }, + "time": "0:01:10.136519", + "diversity": { + "train": { + "mean_centered_corr": -0.019902383908629417, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04721210524439812 + }, + "val": { + "mean_centered_corr": -0.019949622452259064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06361114978790283 + }, + "test": { + "mean_centered_corr": -0.019116314128041267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07747960835695267 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 29, + 19, + 1, + 10, + 25, + 21, + 16 + ], + "head_selection_time": "0:00:00.133304" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4a592983a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24066084712400115, + "mae": 0.1302179992198944, + "r2": 0.6299803256988525, + "score": -0.24066084712400115 + }, + "val": { + "rmse": 0.23897181954001903, + "mae": 0.13393160700798035, + "r2": 0.6185438632965088, + "score": -0.23897181954001903 + }, + "test": { + "rmse": 0.23496733361066027, + "mae": 0.1394679695367813, + "r2": 0.6227279901504517, + "score": -0.23496733361066027 + } + }, + "time": "0:01:11.433192", + "diversity": { + "train": { + "mean_centered_corr": -0.01913526840507984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782585799694061 + }, + "val": { + "mean_centered_corr": -0.01699827052652836, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06600741297006607 + }, + "test": { + "mean_centered_corr": -0.020212531089782715, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08567789196968079 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.016013" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0.toml new file mode 100644 index 000000000..abb0f08fd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..13e36cb86 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2318619145508088, + "mae": 0.12150537222623825, + "r2": 0.6565427184104919, + "score": -0.2318619145508088 + }, + "val": { + "rmse": 0.2357328737026148, + "mae": 0.12986169755458832, + "r2": 0.6288140416145325, + "score": -0.2357328737026148 + }, + "test": { + "rmse": 0.2351772485906963, + "mae": 0.14941059052944183, + "r2": 0.6220536231994629, + "score": -0.2351772485906963 + } + }, + "time": "0:01:11.433192", + "diversity": { + "train": { + "mean_centered_corr": -0.01913526840507984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782585799694061 + }, + "val": { + "mean_centered_corr": -0.01699827052652836, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06600741297006607 + }, + "test": { + "mean_centered_corr": -0.020212531089782715, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08567789196968079 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ec510aaaf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23260143300346486, + "mae": 0.12271209806203842, + "r2": 0.6543483734130859, + "score": -0.23260143300346486 + }, + "val": { + "rmse": 0.2341784288692296, + "mae": 0.1288924217224121, + "r2": 0.6336931586265564, + "score": -0.2341784288692296 + }, + "test": { + "rmse": 0.23454983071534766, + "mae": 0.14984872937202454, + "r2": 0.6240675449371338, + "score": -0.23454983071534766 + } + }, + "time": "0:01:11.433192", + "diversity": { + "train": { + "mean_centered_corr": -0.01913526840507984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782585799694061 + }, + "val": { + "mean_centered_corr": -0.01699827052652836, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06600741297006607 + }, + "test": { + "mean_centered_corr": -0.020212531089782715, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08567789196968079 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8, + 21, + 27, + 19, + 3, + 25, + 4, + 12, + 0, + 18, + 3, + 24 + ], + "head_selection_time": "0:00:00.198152" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0c1814f5c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.22160140516469537, + "mae": 0.12272227555513382, + "r2": 0.6862679719924927, + "score": -0.22160140516469537 + }, + "val": { + "rmse": 0.24058462987099405, + "mae": 0.13246983289718628, + "r2": 0.613377571105957, + "score": -0.24058462987099405 + }, + "test": { + "rmse": 0.2859647081281448, + "mae": 0.2032262533903122, + "r2": 0.44118958711624146, + "score": -0.2859647081281448 + } + }, + "time": "0:01:30.694989", + "diversity": { + "train": { + "mean_centered_corr": -0.0177434291690588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05118265748023987 + }, + "val": { + "mean_centered_corr": -0.01485136803239584, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0833396464586258 + }, + "test": { + "mean_centered_corr": -0.015522093512117863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11537164449691772 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 10 + ], + "head_selection_time": "0:00:00.015392" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0.toml new file mode 100644 index 000000000..7c3998233 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..0eb526983 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.2047007403568582, + "mae": 0.11088933050632477, + "r2": 0.7322973012924194, + "score": -0.2047007403568582 + }, + "val": { + "rmse": 0.2351519581095747, + "mae": 0.12822481989860535, + "r2": 0.6306412220001221, + "score": -0.2351519581095747 + }, + "test": { + "rmse": 0.2408751648791966, + "mae": 0.15764909982681274, + "r2": 0.6035178899765015, + "score": -0.2408751648791966 + } + }, + "time": "0:01:30.694989", + "diversity": { + "train": { + "mean_centered_corr": -0.0177434291690588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05118265748023987 + }, + "val": { + "mean_centered_corr": -0.01485136803239584, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0833396464586258 + }, + "test": { + "mean_centered_corr": -0.015522093512117863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11537164449691772 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9553bbb39 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,705 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20677708372923248, + "mae": 0.11196121573448181, + "r2": 0.7268389463424683, + "score": -0.20677708372923248 + }, + "val": { + "rmse": 0.23316436266516308, + "mae": 0.12652936577796936, + "r2": 0.6368587017059326, + "score": -0.23316436266516308 + }, + "test": { + "rmse": 0.23171057167720172, + "mae": 0.14218251407146454, + "r2": 0.6331138610839844, + "score": -0.23171057167720172 + } + }, + "time": "0:01:30.694989", + "diversity": { + "train": { + "mean_centered_corr": -0.0177434291690588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05118265748023987 + }, + "val": { + "mean_centered_corr": -0.01485136803239584, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0833396464586258 + }, + "test": { + "mean_centered_corr": -0.015522093512117863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11537164449691772 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 10, + 27, + 7, + 1, + 23, + 16, + 29, + 5, + 17, + 7, + 25, + 29, + 20, + 27, + 1, + 21, + 7, + 22 + ], + "head_selection_time": "0:00:00.284355" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..513383bf7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2438513529798656, + "mae": 0.12642574310302734, + "r2": 0.6201044321060181, + "score": -0.2438513529798656 + }, + "val": { + "rmse": 0.23781902006036262, + "mae": 0.13461825251579285, + "r2": 0.6222152709960938, + "score": -0.23781902006036262 + }, + "test": { + "rmse": 0.2470883298795412, + "mae": 0.16668537259101868, + "r2": 0.5828002095222473, + "score": -0.2470883298795412 + } + }, + "time": "0:01:10.745769", + "diversity": { + "train": { + "mean_centered_corr": -0.01934780180454254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04824277386069298 + }, + "val": { + "mean_centered_corr": -0.018993837758898735, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06548074632883072 + }, + "test": { + "mean_centered_corr": -0.019981293007731438, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0883246660232544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015527" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0.toml new file mode 100644 index 000000000..c73212d0c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..0c3eb60a2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23279507911071862, + "mae": 0.12109043449163437, + "r2": 0.6537725925445557, + "score": -0.23279507911071862 + }, + "val": { + "rmse": 0.2353263382433216, + "mae": 0.12844589352607727, + "r2": 0.6300932168960571, + "score": -0.2353263382433216 + }, + "test": { + "rmse": 0.2351845825591179, + "mae": 0.1493937373161316, + "r2": 0.6220300197601318, + "score": -0.2351845825591179 + } + }, + "time": "0:01:10.745769", + "diversity": { + "train": { + "mean_centered_corr": -0.01934780180454254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04824277386069298 + }, + "val": { + "mean_centered_corr": -0.018993837758898735, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06548074632883072 + }, + "test": { + "mean_centered_corr": -0.019981293007731438, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0883246660232544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9bd901fe9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23261467766548827, + "mae": 0.12115577608346939, + "r2": 0.6543089747428894, + "score": -0.23261467766548827 + }, + "val": { + "rmse": 0.2335463179080623, + "mae": 0.127370223402977, + "r2": 0.6356680393218994, + "score": -0.2335463179080623 + }, + "test": { + "rmse": 0.23522276143697368, + "mae": 0.15016508102416992, + "r2": 0.6219072937965393, + "score": -0.23522276143697368 + } + }, + "time": "0:01:10.745769", + "diversity": { + "train": { + "mean_centered_corr": -0.01934780180454254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04824277386069298 + }, + "val": { + "mean_centered_corr": -0.018993837758898735, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06548074632883072 + }, + "test": { + "mean_centered_corr": -0.019981293007731438, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0883246660232544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 19, + 1, + 20, + 25, + 12, + 21, + 25, + 3, + 28, + 19 + ], + "head_selection_time": "0:00:00.177452" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5e826e434 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2314480246211949, + "mae": 0.12086600810289383, + "r2": 0.6577678322792053, + "score": -0.2314480246211949 + }, + "val": { + "rmse": 0.23862914188251422, + "mae": 0.13088369369506836, + "r2": 0.6196370124816895, + "score": -0.23862914188251422 + }, + "test": { + "rmse": 0.23553986357063933, + "mae": 0.14342397451400757, + "r2": 0.6208872199058533, + "score": -0.23553986357063933 + } + }, + "time": "0:01:17.740928", + "diversity": { + "train": { + "mean_centered_corr": -0.019465479999780655, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04585251957178116 + }, + "val": { + "mean_centered_corr": -0.01962343417108059, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06466832011938095 + }, + "test": { + "mean_centered_corr": -0.016597073525190353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08519241958856583 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015359" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0.toml new file mode 100644 index 000000000..c120dab19 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..76187575b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22484523994052877, + "mae": 0.1174779012799263, + "r2": 0.6770157814025879, + "score": -0.22484523994052877 + }, + "val": { + "rmse": 0.2347666170730829, + "mae": 0.1269145905971527, + "r2": 0.6318507194519043, + "score": -0.2347666170730829 + }, + "test": { + "rmse": 0.23399684917333807, + "mae": 0.14828696846961975, + "r2": 0.625838041305542, + "score": -0.23399684917333807 + } + }, + "time": "0:01:17.740928", + "diversity": { + "train": { + "mean_centered_corr": -0.019465479999780655, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04585251957178116 + }, + "val": { + "mean_centered_corr": -0.01962343417108059, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06466832011938095 + }, + "test": { + "mean_centered_corr": -0.016597073525190353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08519241958856583 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..aa2a368e6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,691 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22569541421502587, + "mae": 0.11816450208425522, + "r2": 0.6745686531066895, + "score": -0.22569541421502587 + }, + "val": { + "rmse": 0.23305490182627153, + "mae": 0.1270279735326767, + "r2": 0.6371996402740479, + "score": -0.23305490182627153 + }, + "test": { + "rmse": 0.23732565171059752, + "mae": 0.15259529650211334, + "r2": 0.6151168346405029, + "score": -0.23732565171059752 + } + }, + "time": "0:01:17.740928", + "diversity": { + "train": { + "mean_centered_corr": -0.019465479999780655, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04585251957178116 + }, + "val": { + "mean_centered_corr": -0.01962343417108059, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06466832011938095 + }, + "test": { + "mean_centered_corr": -0.016597073525190353, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08519241958856583 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 25, + 21, + 29 + ], + "head_selection_time": "0:00:00.074319" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b8e2f7b17 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2241888421744036, + "mae": 0.1289592683315277, + "r2": 0.678898811340332, + "score": -0.2241888421744036 + }, + "val": { + "rmse": 0.23918568828057105, + "mae": 0.14043618738651276, + "r2": 0.6178607940673828, + "score": -0.23918568828057105 + }, + "test": { + "rmse": 0.25293341379845147, + "mae": 0.17517992854118347, + "r2": 0.5628283023834229, + "score": -0.25293341379845147 + } + }, + "time": "0:01:18.549217", + "diversity": { + "train": { + "mean_centered_corr": -0.017244398593902588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045789685100317 + }, + "val": { + "mean_centered_corr": -0.019243216142058372, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06452018767595291 + }, + "test": { + "mean_centered_corr": -0.017609506845474243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08646101504564285 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015402" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0.toml new file mode 100644 index 000000000..e8e4c7ed8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..5c0a0b3d6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.220642866499973, + "mae": 0.11938578635454178, + "r2": 0.6889761686325073, + "score": -0.220642866499973 + }, + "val": { + "rmse": 0.2351252944190024, + "mae": 0.12928679585456848, + "r2": 0.6307249069213867, + "score": -0.2351252944190024 + }, + "test": { + "rmse": 0.23697534320527558, + "mae": 0.15434296429157257, + "r2": 0.6162521839141846, + "score": -0.23697534320527558 + } + }, + "time": "0:01:18.549217", + "diversity": { + "train": { + "mean_centered_corr": -0.017244398593902588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045789685100317 + }, + "val": { + "mean_centered_corr": -0.019243216142058372, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06452018767595291 + }, + "test": { + "mean_centered_corr": -0.017609506845474243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08646101504564285 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..f27463361 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,693 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2206414904658828, + "mae": 0.12224060297012329, + "r2": 0.6889800429344177, + "score": -0.2206414904658828 + }, + "val": { + "rmse": 0.2331196304833006, + "mae": 0.12830887734889984, + "r2": 0.6369980573654175, + "score": -0.2331196304833006 + }, + "test": { + "rmse": 0.23280496841127477, + "mae": 0.14530567824840546, + "r2": 0.6296399831771851, + "score": -0.23280496841127477 + } + }, + "time": "0:01:18.549217", + "diversity": { + "train": { + "mean_centered_corr": -0.017244398593902588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045789685100317 + }, + "val": { + "mean_centered_corr": -0.019243216142058372, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06452018767595291 + }, + "test": { + "mean_centered_corr": -0.017609506845474243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08646101504564285 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 8, + 20, + 31, + 19, + 0 + ], + "head_selection_time": "0:00:00.102701" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..175c33ea7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2425666348719968, + "mae": 0.12444926798343658, + "r2": 0.6240968108177185, + "score": -0.2425666348719968 + }, + "val": { + "rmse": 0.23756948853650636, + "mae": 0.13087332248687744, + "r2": 0.6230076551437378, + "score": -0.23756948853650636 + }, + "test": { + "rmse": 0.24224213014777235, + "mae": 0.15561068058013916, + "r2": 0.5990049839019775, + "score": -0.24224213014777235 + } + }, + "time": "0:01:11.515736", + "diversity": { + "train": { + "mean_centered_corr": -0.019360000267624855, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683505743741989 + }, + "val": { + "mean_centered_corr": -0.019574038684368134, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06655079871416092 + }, + "test": { + "mean_centered_corr": -0.01754024624824524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08368789404630661 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015372" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0.toml new file mode 100644 index 000000000..859a25685 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..54e6a2233 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23343200123041558, + "mae": 0.12027841061353683, + "r2": 0.6518754363059998, + "score": -0.23343200123041558 + }, + "val": { + "rmse": 0.23512438339576403, + "mae": 0.12516294419765472, + "r2": 0.6307278275489807, + "score": -0.23512438339576403 + }, + "test": { + "rmse": 0.2316782780837399, + "mae": 0.14233005046844482, + "r2": 0.633216142654419, + "score": -0.2316782780837399 + } + }, + "time": "0:01:11.515736", + "diversity": { + "train": { + "mean_centered_corr": -0.019360000267624855, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683505743741989 + }, + "val": { + "mean_centered_corr": -0.019574038684368134, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06655079871416092 + }, + "test": { + "mean_centered_corr": -0.01754024624824524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08368789404630661 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..905a607c5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,693 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23458808907797565, + "mae": 0.12325569242238998, + "r2": 0.648418664932251, + "score": -0.23458808907797565 + }, + "val": { + "rmse": 0.23291770554949479, + "mae": 0.1261579543352127, + "r2": 0.6376266479492188, + "score": -0.23291770554949479 + }, + "test": { + "rmse": 0.23458472246540202, + "mae": 0.14723080396652222, + "r2": 0.6239557266235352, + "score": -0.23458472246540202 + } + }, + "time": "0:01:11.515736", + "diversity": { + "train": { + "mean_centered_corr": -0.019360000267624855, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683505743741989 + }, + "val": { + "mean_centered_corr": -0.019574038684368134, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06655079871416092 + }, + "test": { + "mean_centered_corr": -0.01754024624824524, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08368789404630661 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 25, + 21, + 6, + 19, + 0 + ], + "head_selection_time": "0:00:00.101988" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4ce56db81 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24039030054595545, + "mae": 0.1237230896949768, + "r2": 0.6308118104934692, + "score": -0.24039030054595545 + }, + "val": { + "rmse": 0.23560357747729638, + "mae": 0.13029368221759796, + "r2": 0.6292210817337036, + "score": -0.23560357747729638 + }, + "test": { + "rmse": 0.2389191702722735, + "mae": 0.15265432000160217, + "r2": 0.6099308729171753, + "score": -0.2389191702722735 + } + }, + "time": "0:01:12.376039", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015348" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml new file mode 100644 index 000000000..aff429e28 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..92245a889 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23278188472143527, + "mae": 0.12037675827741623, + "r2": 0.6538118124008179, + "score": -0.23278188472143527 + }, + "val": { + "rmse": 0.235050071622046, + "mae": 0.12535566091537476, + "r2": 0.6309611797332764, + "score": -0.235050071622046 + }, + "test": { + "rmse": 0.23184680318718037, + "mae": 0.14313000440597534, + "r2": 0.6326823234558105, + "score": -0.23184680318718037 + } + }, + "time": "0:01:12.376039", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0a73544e2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,692 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23573796221874224, + "mae": 0.12117554992437363, + "r2": 0.6449635028839111, + "score": -0.23573796221874224 + }, + "val": { + "rmse": 0.2324412121462238, + "mae": 0.12522703409194946, + "r2": 0.6391078233718872, + "score": -0.2324412121462238 + }, + "test": { + "rmse": 0.23820740696857753, + "mae": 0.1556985229253769, + "r2": 0.6122515201568604, + "score": -0.23820740696857753 + } + }, + "time": "0:01:12.376039", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 1, + 25, + 21 + ], + "head_selection_time": "0:00:00.090212" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1b85fd03e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24104706607058576, + "mae": 0.1307125687599182, + "r2": 0.6287918090820312, + "score": -0.24104706607058576 + }, + "val": { + "rmse": 0.2372450969584346, + "mae": 0.1306145042181015, + "r2": 0.624036431312561, + "score": -0.2372450969584346 + }, + "test": { + "rmse": 0.23726047684878263, + "mae": 0.14689351618289948, + "r2": 0.6153281927108765, + "score": -0.23726047684878263 + } + }, + "time": "0:01:11.064623", + "diversity": { + "train": { + "mean_centered_corr": -0.019136181101202965, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04613374546170235 + }, + "val": { + "mean_centered_corr": -0.01984967105090618, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134215369820595 + }, + "test": { + "mean_centered_corr": -0.0187427569180727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017097413539886 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015268" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0.toml new file mode 100644 index 000000000..46158c91e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..121ee2f43 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23274665867971067, + "mae": 0.1198442280292511, + "r2": 0.653916597366333, + "score": -0.23274665867971067 + }, + "val": { + "rmse": 0.23505585640746507, + "mae": 0.12548410892486572, + "r2": 0.6309430599212646, + "score": -0.23505585640746507 + }, + "test": { + "rmse": 0.23276235981389973, + "mae": 0.14522086083889008, + "r2": 0.6297755241394043, + "score": -0.23276235981389973 + } + }, + "time": "0:01:11.064623", + "diversity": { + "train": { + "mean_centered_corr": -0.019136181101202965, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04613374546170235 + }, + "val": { + "mean_centered_corr": -0.01984967105090618, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134215369820595 + }, + "test": { + "mean_centered_corr": -0.0187427569180727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017097413539886 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7aa242102 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,696 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23595696457409665, + "mae": 0.12094593048095703, + "r2": 0.644303560256958, + "score": -0.23595696457409665 + }, + "val": { + "rmse": 0.2328726380397863, + "mae": 0.12572719156742096, + "r2": 0.6377668380737305, + "score": -0.2328726380397863 + }, + "test": { + "rmse": 0.23808236498886684, + "mae": 0.15540435910224915, + "r2": 0.6126585006713867, + "score": -0.23808236498886684 + } + }, + "time": "0:01:11.064623", + "diversity": { + "train": { + "mean_centered_corr": -0.019136181101202965, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04613374546170235 + }, + "val": { + "mean_centered_corr": -0.01984967105090618, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134215369820595 + }, + "test": { + "mean_centered_corr": -0.0187427569180727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08017097413539886 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 4, + 16, + 1, + 10, + 25, + 21, + 16, + 4 + ], + "head_selection_time": "0:00:00.145331" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..46f33a2a2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24078892072406902, + "mae": 0.12140180170536041, + "r2": 0.6295864582061768, + "score": -0.24078892072406902 + }, + "val": { + "rmse": 0.23656618722639788, + "mae": 0.1301935315132141, + "r2": 0.6261851191520691, + "score": -0.23656618722639788 + }, + "test": { + "rmse": 0.2443526301182123, + "mae": 0.15885169804096222, + "r2": 0.5919873714447021, + "score": -0.2443526301182123 + } + }, + "time": "0:01:10.820742", + "diversity": { + "train": { + "mean_centered_corr": -0.019902383908629417, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04721210524439812 + }, + "val": { + "mean_centered_corr": -0.019949622452259064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06361114978790283 + }, + "test": { + "mean_centered_corr": -0.019116314128041267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07747960835695267 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015547" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0.toml new file mode 100644 index 000000000..4e30093ec --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..ac383e963 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23225320578180678, + "mae": 0.12026127427816391, + "r2": 0.655382513999939, + "score": -0.23225320578180678 + }, + "val": { + "rmse": 0.23504784483792887, + "mae": 0.12635678052902222, + "r2": 0.6309682130813599, + "score": -0.23504784483792887 + }, + "test": { + "rmse": 0.23465037063607563, + "mae": 0.14887936413288116, + "r2": 0.6237452030181885, + "score": -0.23465037063607563 + } + }, + "time": "0:01:10.820742", + "diversity": { + "train": { + "mean_centered_corr": -0.019902383908629417, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04721210524439812 + }, + "val": { + "mean_centered_corr": -0.019949622452259064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06361114978790283 + }, + "test": { + "mean_centered_corr": -0.019116314128041267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07747960835695267 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..64930bc5e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,695 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2344874747922904, + "mae": 0.1211647242307663, + "r2": 0.6487202048301697, + "score": -0.2344874747922904 + }, + "val": { + "rmse": 0.23334387616340208, + "mae": 0.12681756913661957, + "r2": 0.6362993717193604, + "score": -0.23334387616340208 + }, + "test": { + "rmse": 0.2380340654227449, + "mae": 0.15519878268241882, + "r2": 0.6128156185150146, + "score": -0.2380340654227449 + } + }, + "time": "0:01:10.820742", + "diversity": { + "train": { + "mean_centered_corr": -0.019902383908629417, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04721210524439812 + }, + "val": { + "mean_centered_corr": -0.019949622452259064, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06361114978790283 + }, + "test": { + "mean_centered_corr": -0.019116314128041267, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07747960835695267 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 29, + 19, + 1, + 10, + 25, + 21, + 16 + ], + "head_selection_time": "0:00:00.131017" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1db8be0ad --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24066084712400115, + "mae": 0.1302179992198944, + "r2": 0.6299803256988525, + "score": -0.24066084712400115 + }, + "val": { + "rmse": 0.23897181954001903, + "mae": 0.13393160700798035, + "r2": 0.6185438632965088, + "score": -0.23897181954001903 + }, + "test": { + "rmse": 0.23496733361066027, + "mae": 0.1394679695367813, + "r2": 0.6227279901504517, + "score": -0.23496733361066027 + } + }, + "time": "0:01:10.329042", + "diversity": { + "train": { + "mean_centered_corr": -0.01913526840507984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782585799694061 + }, + "val": { + "mean_centered_corr": -0.01699827052652836, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06600741297006607 + }, + "test": { + "mean_centered_corr": -0.020212531089782715, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08567789196968079 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.015400" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0.toml new file mode 100644 index 000000000..bcd7aae35 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..3649d701d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2318619145508088, + "mae": 0.12150537222623825, + "r2": 0.6565427184104919, + "score": -0.2318619145508088 + }, + "val": { + "rmse": 0.2357328737026148, + "mae": 0.12986169755458832, + "r2": 0.6288140416145325, + "score": -0.2357328737026148 + }, + "test": { + "rmse": 0.2351772485906963, + "mae": 0.14941059052944183, + "r2": 0.6220536231994629, + "score": -0.2351772485906963 + } + }, + "time": "0:01:10.329042", + "diversity": { + "train": { + "mean_centered_corr": -0.01913526840507984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782585799694061 + }, + "val": { + "mean_centered_corr": -0.01699827052652836, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06600741297006607 + }, + "test": { + "mean_centered_corr": -0.020212531089782715, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08567789196968079 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1ceba83b0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23260143300346486, + "mae": 0.12271209806203842, + "r2": 0.6543483734130859, + "score": -0.23260143300346486 + }, + "val": { + "rmse": 0.2341784288692296, + "mae": 0.1288924217224121, + "r2": 0.6336931586265564, + "score": -0.2341784288692296 + }, + "test": { + "rmse": 0.23454983071534766, + "mae": 0.14984872937202454, + "r2": 0.6240675449371338, + "score": -0.23454983071534766 + } + }, + "time": "0:01:10.329042", + "diversity": { + "train": { + "mean_centered_corr": -0.01913526840507984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782585799694061 + }, + "val": { + "mean_centered_corr": -0.01699827052652836, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06600741297006607 + }, + "test": { + "mean_centered_corr": -0.020212531089782715, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08567789196968079 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8, + 21, + 27, + 19, + 3, + 25, + 4, + 12, + 0, + 18, + 3, + 24 + ], + "head_selection_time": "0:00:00.196120" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..44a38d101 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.22160140516469537, + "mae": 0.12272227555513382, + "r2": 0.6862679719924927, + "score": -0.22160140516469537 + }, + "val": { + "rmse": 0.24058462987099405, + "mae": 0.13246983289718628, + "r2": 0.613377571105957, + "score": -0.24058462987099405 + }, + "test": { + "rmse": 0.2859647081281448, + "mae": 0.2032262533903122, + "r2": 0.44118958711624146, + "score": -0.2859647081281448 + } + }, + "time": "0:01:30.096626", + "diversity": { + "train": { + "mean_centered_corr": -0.0177434291690588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05118265748023987 + }, + "val": { + "mean_centered_corr": -0.01485136803239584, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0833396464586258 + }, + "test": { + "mean_centered_corr": -0.015522093512117863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11537164449691772 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 10 + ], + "head_selection_time": "0:00:00.015160" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0.toml new file mode 100644 index 000000000..2724d4773 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..85c64d585 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.2047007403568582, + "mae": 0.11088933050632477, + "r2": 0.7322973012924194, + "score": -0.2047007403568582 + }, + "val": { + "rmse": 0.2351519581095747, + "mae": 0.12822481989860535, + "r2": 0.6306412220001221, + "score": -0.2351519581095747 + }, + "test": { + "rmse": 0.2408751648791966, + "mae": 0.15764909982681274, + "r2": 0.6035178899765015, + "score": -0.2408751648791966 + } + }, + "time": "0:01:30.096626", + "diversity": { + "train": { + "mean_centered_corr": -0.0177434291690588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05118265748023987 + }, + "val": { + "mean_centered_corr": -0.01485136803239584, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0833396464586258 + }, + "test": { + "mean_centered_corr": -0.015522093512117863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11537164449691772 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4f904616f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,705 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20677708372923248, + "mae": 0.11196121573448181, + "r2": 0.7268389463424683, + "score": -0.20677708372923248 + }, + "val": { + "rmse": 0.23316436266516308, + "mae": 0.12652936577796936, + "r2": 0.6368587017059326, + "score": -0.23316436266516308 + }, + "test": { + "rmse": 0.23171057167720172, + "mae": 0.14218251407146454, + "r2": 0.6331138610839844, + "score": -0.23171057167720172 + } + }, + "time": "0:01:30.096626", + "diversity": { + "train": { + "mean_centered_corr": -0.0177434291690588, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05118265748023987 + }, + "val": { + "mean_centered_corr": -0.01485136803239584, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0833396464586258 + }, + "test": { + "mean_centered_corr": -0.015522093512117863, + "mean_pairwise_disagreement": NaN, + "member_std": 0.11537164449691772 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 10, + 27, + 7, + 1, + 23, + 16, + 29, + 5, + 17, + 7, + 25, + 29, + 20, + 27, + 1, + 21, + 7, + 22 + ], + "head_selection_time": "0:00:00.279741" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..0e1f8f255 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2438513529798656, + "mae": 0.12642574310302734, + "r2": 0.6201044321060181, + "score": -0.2438513529798656 + }, + "val": { + "rmse": 0.23781902006036262, + "mae": 0.13461825251579285, + "r2": 0.6222152709960938, + "score": -0.23781902006036262 + }, + "test": { + "rmse": 0.2470883298795412, + "mae": 0.16668537259101868, + "r2": 0.5828002095222473, + "score": -0.2470883298795412 + } + }, + "time": "0:01:11.258846", + "diversity": { + "train": { + "mean_centered_corr": -0.01934780180454254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04824277386069298 + }, + "val": { + "mean_centered_corr": -0.018993837758898735, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06548074632883072 + }, + "test": { + "mean_centered_corr": -0.019981293007731438, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0883246660232544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015208" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0.toml new file mode 100644 index 000000000..65c4d5ef3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..32c548ccb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23279507911071862, + "mae": 0.12109043449163437, + "r2": 0.6537725925445557, + "score": -0.23279507911071862 + }, + "val": { + "rmse": 0.2353263382433216, + "mae": 0.12844589352607727, + "r2": 0.6300932168960571, + "score": -0.2353263382433216 + }, + "test": { + "rmse": 0.2351845825591179, + "mae": 0.1493937373161316, + "r2": 0.6220300197601318, + "score": -0.2351845825591179 + } + }, + "time": "0:01:11.258846", + "diversity": { + "train": { + "mean_centered_corr": -0.01934780180454254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04824277386069298 + }, + "val": { + "mean_centered_corr": -0.018993837758898735, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06548074632883072 + }, + "test": { + "mean_centered_corr": -0.019981293007731438, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0883246660232544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2d7b9734f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23261467766548827, + "mae": 0.12115577608346939, + "r2": 0.6543089747428894, + "score": -0.23261467766548827 + }, + "val": { + "rmse": 0.2335463179080623, + "mae": 0.127370223402977, + "r2": 0.6356680393218994, + "score": -0.2335463179080623 + }, + "test": { + "rmse": 0.23522276143697368, + "mae": 0.15016508102416992, + "r2": 0.6219072937965393, + "score": -0.23522276143697368 + } + }, + "time": "0:01:11.258846", + "diversity": { + "train": { + "mean_centered_corr": -0.01934780180454254, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04824277386069298 + }, + "val": { + "mean_centered_corr": -0.018993837758898735, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06548074632883072 + }, + "test": { + "mean_centered_corr": -0.019981293007731438, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0883246660232544 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 19, + 1, + 20, + 25, + 12, + 21, + 25, + 3, + 28, + 19 + ], + "head_selection_time": "0:00:00.174015" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b4357867a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24166333481256672, + "mae": 0.1247403547167778, + "r2": 0.6268912553787231, + "score": -0.24166333481256672 + }, + "val": { + "rmse": 0.23872630190199182, + "mae": 0.12959358096122742, + "r2": 0.6193272471427917, + "score": -0.23872630190199182 + }, + "test": { + "rmse": 0.23512648270487807, + "mae": 0.1454058140516281, + "r2": 0.6222167611122131, + "score": -0.23512648270487807 + } + }, + "time": "0:01:32.555768", + "diversity": { + "train": { + "mean_centered_corr": -0.01866377331316471, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04515796899795532 + }, + "val": { + "mean_centered_corr": -0.019534021615982056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059918854385614395 + }, + "test": { + "mean_centered_corr": -0.016633860766887665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0777917131781578 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015199" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0.toml new file mode 100644 index 000000000..efb5a9c0b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..5024a184c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23281516131217675, + "mae": 0.12193623930215836, + "r2": 0.653712809085846, + "score": -0.23281516131217675 + }, + "val": { + "rmse": 0.23518726739965692, + "mae": 0.12629030644893646, + "r2": 0.6305302381515503, + "score": -0.23518726739965692 + }, + "test": { + "rmse": 0.23268638913916373, + "mae": 0.14553509652614594, + "r2": 0.6300171613693237, + "score": -0.23268638913916373 + } + }, + "time": "0:01:32.555768", + "diversity": { + "train": { + "mean_centered_corr": -0.01866377331316471, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04515796899795532 + }, + "val": { + "mean_centered_corr": -0.019534021615982056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059918854385614395 + }, + "test": { + "mean_centered_corr": -0.016633860766887665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0777917131781578 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ec2e59f69 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,700 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2349319119810031, + "mae": 0.12289345264434814, + "r2": 0.6473873257637024, + "score": -0.2349319119810031 + }, + "val": { + "rmse": 0.23347937804489072, + "mae": 0.12790052592754364, + "r2": 0.6358768343925476, + "score": -0.23347937804489072 + }, + "test": { + "rmse": 0.24206068964408525, + "mae": 0.16086436808109283, + "r2": 0.5996055006980896, + "score": -0.24206068964408525 + } + }, + "time": "0:01:32.555768", + "diversity": { + "train": { + "mean_centered_corr": -0.01866377331316471, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04515796899795532 + }, + "val": { + "mean_centered_corr": -0.019534021615982056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.059918854385614395 + }, + "test": { + "mean_centered_corr": -0.016633860766887665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0777917131781578 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 25, + 21, + 1, + 19, + 25, + 10 + ], + "head_selection_time": "0:00:00.116290" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a0a0afcdb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24294329966861838, + "mae": 0.12473063915967941, + "r2": 0.6229285001754761, + "score": -0.24294329966861838 + }, + "val": { + "rmse": 0.23800428115225924, + "mae": 0.13076891005039215, + "r2": 0.6216264367103577, + "score": -0.23800428115225924 + }, + "test": { + "rmse": 0.23509819987146183, + "mae": 0.1425461322069168, + "r2": 0.6223076581954956, + "score": -0.23509819987146183 + } + }, + "time": "0:01:32.973453", + "diversity": { + "train": { + "mean_centered_corr": -0.01892852410674095, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046223822981119156 + }, + "val": { + "mean_centered_corr": -0.019010689109563828, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061593979597091675 + }, + "test": { + "mean_centered_corr": -0.01590883731842041, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07384058833122253 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015239" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0.toml new file mode 100644 index 000000000..4c913eb52 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..193effd39 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23234710773024822, + "mae": 0.12129862606525421, + "r2": 0.6551038026809692, + "score": -0.23234710773024822 + }, + "val": { + "rmse": 0.23483364211643792, + "mae": 0.12628823518753052, + "r2": 0.6316404938697815, + "score": -0.23483364211643792 + }, + "test": { + "rmse": 0.23156642561711185, + "mae": 0.14394673705101013, + "r2": 0.6335701942443848, + "score": -0.23156642561711185 + } + }, + "time": "0:01:32.973453", + "diversity": { + "train": { + "mean_centered_corr": -0.01892852410674095, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046223822981119156 + }, + "val": { + "mean_centered_corr": -0.019010689109563828, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061593979597091675 + }, + "test": { + "mean_centered_corr": -0.01590883731842041, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07384058833122253 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9b58999fc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,702 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23416801690807423, + "mae": 0.12230122834444046, + "r2": 0.6496766805648804, + "score": -0.23416801690807423 + }, + "val": { + "rmse": 0.23277299468998114, + "mae": 0.12583908438682556, + "r2": 0.6380767822265625, + "score": -0.23277299468998114 + }, + "test": { + "rmse": 0.2357753957826797, + "mae": 0.15136900544166565, + "r2": 0.6201286315917969, + "score": -0.2357753957826797 + } + }, + "time": "0:01:32.973453", + "diversity": { + "train": { + "mean_centered_corr": -0.01892852410674095, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046223822981119156 + }, + "val": { + "mean_centered_corr": -0.019010689109563828, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061593979597091675 + }, + "test": { + "mean_centered_corr": -0.01590883731842041, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07384058833122253 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 6, + 0, + 19, + 1, + 25, + 21, + 25, + 1 + ], + "head_selection_time": "0:00:00.144542" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b425a37ad --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21889456161617807, + "mae": 0.12161961197853088, + "r2": 0.6938855648040771, + "score": -0.21889456161617807 + }, + "val": { + "rmse": 0.23798911368843165, + "mae": 0.13175897300243378, + "r2": 0.621674656867981, + "score": -0.23798911368843165 + }, + "test": { + "rmse": 0.24638280846527752, + "mae": 0.16409903764724731, + "r2": 0.585179328918457, + "score": -0.24638280846527752 + } + }, + "time": "0:01:41.397939", + "diversity": { + "train": { + "mean_centered_corr": -0.01843600906431675, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046984437853097916 + }, + "val": { + "mean_centered_corr": -0.01927182450890541, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06631730496883392 + }, + "test": { + "mean_centered_corr": -0.018402421846985817, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08607663214206696 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015360" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0.toml new file mode 100644 index 000000000..b700865b4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..96e694441 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2200473901522814, + "mae": 0.11505363136529922, + "r2": 0.6906527280807495, + "score": -0.2200473901522814 + }, + "val": { + "rmse": 0.23457506700663386, + "mae": 0.12505945563316345, + "r2": 0.6324512362480164, + "score": -0.23457506700663386 + }, + "test": { + "rmse": 0.2318295295556057, + "mae": 0.1426994651556015, + "r2": 0.6327370405197144, + "score": -0.2318295295556057 + } + }, + "time": "0:01:41.397939", + "diversity": { + "train": { + "mean_centered_corr": -0.01843600906431675, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046984437853097916 + }, + "val": { + "mean_centered_corr": -0.01927182450890541, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06631730496883392 + }, + "test": { + "mean_centered_corr": -0.018402421846985817, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08607663214206696 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..580711c62 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,697 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22139641227752963, + "mae": 0.12048649042844772, + "r2": 0.6868481040000916, + "score": -0.22139641227752963 + }, + "val": { + "rmse": 0.2329412795410917, + "mae": 0.12749527394771576, + "r2": 0.637553334236145, + "score": -0.2329412795410917 + }, + "test": { + "rmse": 0.23349850795747007, + "mae": 0.14457938075065613, + "r2": 0.6274300813674927, + "score": -0.23349850795747007 + } + }, + "time": "0:01:41.397939", + "diversity": { + "train": { + "mean_centered_corr": -0.01843600906431675, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046984437853097916 + }, + "val": { + "mean_centered_corr": -0.01927182450890541, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06631730496883392 + }, + "test": { + "mean_centered_corr": -0.018402421846985817, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08607663214206696 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 21, + 8, + 31, + 0 + ], + "head_selection_time": "0:00:00.074215" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..83b9d9524 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2248883132434097, + "mae": 0.12571877241134644, + "r2": 0.6768920421600342, + "score": -0.2248883132434097 + }, + "val": { + "rmse": 0.23993622491293543, + "mae": 0.13967159390449524, + "r2": 0.6154587864875793, + "score": -0.23993622491293543 + }, + "test": { + "rmse": 0.2477713353601464, + "mae": 0.16416500508785248, + "r2": 0.5804905891418457, + "score": -0.2477713353601464 + } + }, + "time": "0:01:42.023156", + "diversity": { + "train": { + "mean_centered_corr": -0.017943480983376503, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04725648835301399 + }, + "val": { + "mean_centered_corr": -0.017995651811361313, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07173214852809906 + }, + "test": { + "mean_centered_corr": -0.018061438575387, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10140956938266754 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015426" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0.toml new file mode 100644 index 000000000..46394825d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..4f69b08da --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.217070278259022, + "mae": 0.11539068818092346, + "r2": 0.6989666223526001, + "score": -0.217070278259022 + }, + "val": { + "rmse": 0.2351439260504182, + "mae": 0.1256217062473297, + "r2": 0.6306663751602173, + "score": -0.2351439260504182 + }, + "test": { + "rmse": 0.23458613581335708, + "mae": 0.1473771333694458, + "r2": 0.6239511370658875, + "score": -0.23458613581335708 + } + }, + "time": "0:01:42.023156", + "diversity": { + "train": { + "mean_centered_corr": -0.017943480983376503, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04725648835301399 + }, + "val": { + "mean_centered_corr": -0.017995651811361313, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07173214852809906 + }, + "test": { + "mean_centered_corr": -0.018061438575387, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10140956938266754 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..dc63bb828 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22117446261892904, + "mae": 0.11840678006410599, + "r2": 0.6874756813049316, + "score": -0.22117446261892904 + }, + "val": { + "rmse": 0.23356126348868944, + "mae": 0.12733577191829681, + "r2": 0.6356213688850403, + "score": -0.23356126348868944 + }, + "test": { + "rmse": 0.2502618550799444, + "mae": 0.168344646692276, + "r2": 0.5720146894454956, + "score": -0.2502618550799444 + } + }, + "time": "0:01:42.023156", + "diversity": { + "train": { + "mean_centered_corr": -0.017943480983376503, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04725648835301399 + }, + "val": { + "mean_centered_corr": -0.017995651811361313, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07173214852809906 + }, + "test": { + "mean_centered_corr": -0.018061438575387, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10140956938266754 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 25, + 0, + 19, + 10 + ], + "head_selection_time": "0:00:00.088092" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4238dea50 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24031171096652382, + "mae": 0.12308599799871445, + "r2": 0.6310532093048096, + "score": -0.24031171096652382 + }, + "val": { + "rmse": 0.2360115531890879, + "mae": 0.13377444446086884, + "r2": 0.6279358863830566, + "score": -0.2360115531890879 + }, + "test": { + "rmse": 0.23473422019733492, + "mae": 0.14439672231674194, + "r2": 0.6234762668609619, + "score": -0.23473422019733492 + } + }, + "time": "0:01:30.869624", + "diversity": { + "train": { + "mean_centered_corr": -0.01947913132607937, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0460021011531353 + }, + "val": { + "mean_centered_corr": -0.0197152066975832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061192721128463745 + }, + "test": { + "mean_centered_corr": -0.017801720649003983, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07759672403335571 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015163" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0.toml new file mode 100644 index 000000000..ad5a60008 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..108a0a152 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2315330821822192, + "mae": 0.11987455189228058, + "r2": 0.6575162410736084, + "score": -0.2315330821822192 + }, + "val": { + "rmse": 0.2346577290091983, + "mae": 0.12638406455516815, + "r2": 0.6321921348571777, + "score": -0.2346577290091983 + }, + "test": { + "rmse": 0.23333821657015275, + "mae": 0.1463044285774231, + "r2": 0.6279414296150208, + "score": -0.23333821657015275 + } + }, + "time": "0:01:30.869624", + "diversity": { + "train": { + "mean_centered_corr": -0.01947913132607937, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0460021011531353 + }, + "val": { + "mean_centered_corr": -0.0197152066975832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061192721128463745 + }, + "test": { + "mean_centered_corr": -0.017801720649003983, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07759672403335571 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..0969748b8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23331299019728063, + "mae": 0.12291000783443451, + "r2": 0.6522302627563477, + "score": -0.23331299019728063 + }, + "val": { + "rmse": 0.23283208982244355, + "mae": 0.12645916640758514, + "r2": 0.6378929615020752, + "score": -0.23283208982244355 + }, + "test": { + "rmse": 0.23176779995051228, + "mae": 0.1443839818239212, + "r2": 0.6329326629638672, + "score": -0.23176779995051228 + } + }, + "time": "0:01:30.869624", + "diversity": { + "train": { + "mean_centered_corr": -0.01947913132607937, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0460021011531353 + }, + "val": { + "mean_centered_corr": -0.0197152066975832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.061192721128463745 + }, + "test": { + "mean_centered_corr": -0.017801720649003983, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07759672403335571 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 16, + 0, + 21, + 1, + 19 + ], + "head_selection_time": "0:00:00.101259" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3a53f5dea --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2411780624535083, + "mae": 0.12449400126934052, + "r2": 0.6283881664276123, + "score": -0.2411780624535083 + }, + "val": { + "rmse": 0.23637143045682502, + "mae": 0.13214340806007385, + "r2": 0.6268003582954407, + "score": -0.23637143045682502 + }, + "test": { + "rmse": 0.2444339973181191, + "mae": 0.1604158580303192, + "r2": 0.5917155742645264, + "score": -0.2444339973181191 + } + }, + "time": "0:01:30.058806", + "diversity": { + "train": { + "mean_centered_corr": -0.01962215080857277, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04651423916220665 + }, + "val": { + "mean_centered_corr": -0.019082706421613693, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06017685681581497 + }, + "test": { + "mean_centered_corr": -0.01942536234855652, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08030469715595245 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015428" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0.toml new file mode 100644 index 000000000..eca59c06e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..eea0cd4ec --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23221441033396834, + "mae": 0.12085375189781189, + "r2": 0.6554976105690002, + "score": -0.23221441033396834 + }, + "val": { + "rmse": 0.23515957801466283, + "mae": 0.1254815012216568, + "r2": 0.6306172609329224, + "score": -0.23515957801466283 + }, + "test": { + "rmse": 0.23255229143999848, + "mae": 0.14425288140773773, + "r2": 0.6304435133934021, + "score": -0.23255229143999848 + } + }, + "time": "0:01:30.058806", + "diversity": { + "train": { + "mean_centered_corr": -0.01962215080857277, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04651423916220665 + }, + "val": { + "mean_centered_corr": -0.019082706421613693, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06017685681581497 + }, + "test": { + "mean_centered_corr": -0.01942536234855652, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08030469715595245 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..9432dce69 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,701 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23465978486724487, + "mae": 0.12119866162538528, + "r2": 0.6482037305831909, + "score": -0.23465978486724487 + }, + "val": { + "rmse": 0.233185579288438, + "mae": 0.12584175169467926, + "r2": 0.6367926597595215, + "score": -0.233185579288438 + }, + "test": { + "rmse": 0.23514442509167285, + "mae": 0.15004310011863708, + "r2": 0.6221591234207153, + "score": -0.23514442509167285 + } + }, + "time": "0:01:30.058806", + "diversity": { + "train": { + "mean_centered_corr": -0.01962215080857277, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04651423916220665 + }, + "val": { + "mean_centered_corr": -0.019082706421613693, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06017685681581497 + }, + "test": { + "mean_centered_corr": -0.01942536234855652, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08030469715595245 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 25, + 1, + 28, + 16, + 21, + 6 + ], + "head_selection_time": "0:00:00.131699" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6deb7eeed --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24152145784724474, + "mae": 0.12505307793617249, + "r2": 0.6273292303085327, + "score": -0.24152145784724474 + }, + "val": { + "rmse": 0.2382472279758646, + "mae": 0.13743972778320312, + "r2": 0.6208535432815552, + "score": -0.2382472279758646 + }, + "test": { + "rmse": 0.2416035703731357, + "mae": 0.15706877410411835, + "r2": 0.6011162996292114, + "score": -0.2416035703731357 + } + }, + "time": "0:01:33.524887", + "diversity": { + "train": { + "mean_centered_corr": -0.018592942506074905, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046543560922145844 + }, + "val": { + "mean_centered_corr": -0.01946503296494484, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134062260389328 + }, + "test": { + "mean_centered_corr": -0.01856323331594467, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08496059477329254 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015238" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0.toml new file mode 100644 index 000000000..091455f42 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..9cef9a45b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2314695997411829, + "mae": 0.12160827219486237, + "r2": 0.6577039957046509, + "score": -0.2314695997411829 + }, + "val": { + "rmse": 0.2349283758671495, + "mae": 0.12933029234409332, + "r2": 0.6313432455062866, + "score": -0.2349283758671495 + }, + "test": { + "rmse": 0.23427005643589985, + "mae": 0.1490129977464676, + "r2": 0.6249638199806213, + "score": -0.23427005643589985 + } + }, + "time": "0:01:33.524887", + "diversity": { + "train": { + "mean_centered_corr": -0.018592942506074905, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046543560922145844 + }, + "val": { + "mean_centered_corr": -0.01946503296494484, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134062260389328 + }, + "test": { + "mean_centered_corr": -0.01856323331594467, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08496059477329254 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..72a253d60 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,703 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2324655315526619, + "mae": 0.1217086911201477, + "r2": 0.6547521352767944, + "score": -0.2324655315526619 + }, + "val": { + "rmse": 0.23350649291605036, + "mae": 0.12837612628936768, + "r2": 0.6357922554016113, + "score": -0.23350649291605036 + }, + "test": { + "rmse": 0.23477688348213874, + "mae": 0.14960812032222748, + "r2": 0.6233393549919128, + "score": -0.23477688348213874 + } + }, + "time": "0:01:33.524887", + "diversity": { + "train": { + "mean_centered_corr": -0.018592942506074905, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046543560922145844 + }, + "val": { + "mean_centered_corr": -0.01946503296494484, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06134062260389328 + }, + "test": { + "mean_centered_corr": -0.01856323331594467, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08496059477329254 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 19, + 20, + 3, + 29, + 25, + 12, + 27, + 25, + 10 + ], + "head_selection_time": "0:00:00.158843" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..170597da7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24317591994456592, + "mae": 0.12499266862869263, + "r2": 0.6222060322761536, + "score": -0.24317591994456592 + }, + "val": { + "rmse": 0.23764755063810214, + "mae": 0.13410833477973938, + "r2": 0.6227598190307617, + "score": -0.23764755063810214 + }, + "test": { + "rmse": 0.24545806951373206, + "mae": 0.16299617290496826, + "r2": 0.588287353515625, + "score": -0.24545806951373206 + } + }, + "time": "0:01:30.594432", + "diversity": { + "train": { + "mean_centered_corr": -0.01936914771795273, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0473526269197464 + }, + "val": { + "mean_centered_corr": -0.019165920093655586, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06515220552682877 + }, + "test": { + "mean_centered_corr": -0.020623264834284782, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08508463948965073 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015365" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0.toml new file mode 100644 index 000000000..0b70f916d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..7d43cf48c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23234297110267305, + "mae": 0.12101900577545166, + "r2": 0.655116081237793, + "score": -0.23234297110267305 + }, + "val": { + "rmse": 0.23481945176098018, + "mae": 0.12734994292259216, + "r2": 0.6316850185394287, + "score": -0.23481945176098018 + }, + "test": { + "rmse": 0.23330914213736356, + "mae": 0.1461166888475418, + "r2": 0.6280341148376465, + "score": -0.23330914213736356 + } + }, + "time": "0:01:30.594432", + "diversity": { + "train": { + "mean_centered_corr": -0.01936914771795273, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0473526269197464 + }, + "val": { + "mean_centered_corr": -0.019165920093655586, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06515220552682877 + }, + "test": { + "mean_centered_corr": -0.020623264834284782, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08508463948965073 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..62ec5b8fc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23521274413986984, + "mae": 0.12193334102630615, + "r2": 0.6465438008308411, + "score": -0.23521274413986984 + }, + "val": { + "rmse": 0.23351058500178656, + "mae": 0.12763343751430511, + "r2": 0.6357795000076294, + "score": -0.23351058500178656 + }, + "test": { + "rmse": 0.23619933599335108, + "mae": 0.15221840143203735, + "r2": 0.6187613010406494, + "score": -0.23619933599335108 + } + }, + "time": "0:01:30.594432", + "diversity": { + "train": { + "mean_centered_corr": -0.01936914771795273, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0473526269197464 + }, + "val": { + "mean_centered_corr": -0.019165920093655586, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06515220552682877 + }, + "test": { + "mean_centered_corr": -0.020623264834284782, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08508463948965073 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 19, + 10, + 25, + 3 + ], + "head_selection_time": "0:00:00.087579" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7a0c87fb6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.23319688178416265, + "mae": 0.12308622896671295, + "r2": 0.6525763273239136, + "score": -0.23319688178416265 + }, + "val": { + "rmse": 0.2388627040189618, + "mae": 0.13744254410266876, + "r2": 0.6188920736312866, + "score": -0.2388627040189618 + }, + "test": { + "rmse": 0.24858429278757524, + "mae": 0.16933724284172058, + "r2": 0.5777331590652466, + "score": -0.24858429278757524 + } + }, + "time": "0:01:40.878188", + "diversity": { + "train": { + "mean_centered_corr": -0.01871233992278576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815390333533287 + }, + "val": { + "mean_centered_corr": -0.01693091168999672, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06701929122209549 + }, + "test": { + "mean_centered_corr": -0.018649011850357056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09262414276599884 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015240" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0.toml new file mode 100644 index 000000000..0567ba4d7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..b9a89f1af --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22028603428626464, + "mae": 0.11624135076999664, + "r2": 0.6899813413619995, + "score": -0.22028603428626464 + }, + "val": { + "rmse": 0.235230434477205, + "mae": 0.12826579809188843, + "r2": 0.6303946375846863, + "score": -0.235230434477205 + }, + "test": { + "rmse": 0.23369840873687428, + "mae": 0.14666512608528137, + "r2": 0.6267918348312378, + "score": -0.23369840873687428 + } + }, + "time": "0:01:40.878188", + "diversity": { + "train": { + "mean_centered_corr": -0.01871233992278576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815390333533287 + }, + "val": { + "mean_centered_corr": -0.01693091168999672, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06701929122209549 + }, + "test": { + "mean_centered_corr": -0.018649011850357056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09262414276599884 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..a37b48623 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,708 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22111136733588857, + "mae": 0.11676743626594543, + "r2": 0.6876538991928101, + "score": -0.22111136733588857 + }, + "val": { + "rmse": 0.23403326393093224, + "mae": 0.12714654207229614, + "r2": 0.6341471672058105, + "score": -0.23403326393093224 + }, + "test": { + "rmse": 0.23267639873508839, + "mae": 0.1456754505634308, + "r2": 0.630048930644989, + "score": -0.23267639873508839 + } + }, + "time": "0:01:40.878188", + "diversity": { + "train": { + "mean_centered_corr": -0.01871233992278576, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04815390333533287 + }, + "val": { + "mean_centered_corr": -0.01693091168999672, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06701929122209549 + }, + "test": { + "mean_centered_corr": -0.018649011850357056, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09262414276599884 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 19, + 10, + 23, + 26, + 12, + 22, + 17, + 20, + 28, + 8, + 21, + 23, + 22, + 19 + ], + "head_selection_time": "0:00:00.235823" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..180ceb2b4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23904784974230414, + "mae": 0.12625300884246826, + "r2": 0.6349237561225891, + "score": -0.23904784974230414 + }, + "val": { + "rmse": 0.23805604517030857, + "mae": 0.13564030826091766, + "r2": 0.621461808681488, + "score": -0.23805604517030857 + }, + "test": { + "rmse": 0.24432422600723777, + "mae": 0.15747275948524475, + "r2": 0.5920822024345398, + "score": -0.24432422600723777 + } + }, + "time": "0:01:31.232910", + "diversity": { + "train": { + "mean_centered_corr": -0.019181683659553528, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046132441610097885 + }, + "val": { + "mean_centered_corr": -0.019297907128930092, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06382269412279129 + }, + "test": { + "mean_centered_corr": -0.01942778192460537, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07468391209840775 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.015153" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0.toml new file mode 100644 index 000000000..f027505c7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..f67da74d3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23172767733857508, + "mae": 0.12039440870285034, + "r2": 0.6569403409957886, + "score": -0.23172767733857508 + }, + "val": { + "rmse": 0.2350451108570483, + "mae": 0.1278354525566101, + "r2": 0.6309767961502075, + "score": -0.2350451108570483 + }, + "test": { + "rmse": 0.23328519811438406, + "mae": 0.14723573625087738, + "r2": 0.6281104683876038, + "score": -0.23328519811438406 + } + }, + "time": "0:01:31.232910", + "diversity": { + "train": { + "mean_centered_corr": -0.019181683659553528, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046132441610097885 + }, + "val": { + "mean_centered_corr": -0.019297907128930092, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06382269412279129 + }, + "test": { + "mean_centered_corr": -0.01942778192460537, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07468391209840775 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..550a35e9a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,703 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2334628076252813, + "mae": 0.12108320742845535, + "r2": 0.6517835259437561, + "score": -0.2334628076252813 + }, + "val": { + "rmse": 0.23337935525306813, + "mae": 0.12764780223369598, + "r2": 0.6361887454986572, + "score": -0.23337935525306813 + }, + "test": { + "rmse": 0.2363605083000373, + "mae": 0.1529415249824524, + "r2": 0.6182408332824707, + "score": -0.2363605083000373 + } + }, + "time": "0:01:31.232910", + "diversity": { + "train": { + "mean_centered_corr": -0.019181683659553528, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046132441610097885 + }, + "val": { + "mean_centered_corr": -0.019297907128930092, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06382269412279129 + }, + "test": { + "mean_centered_corr": -0.01942778192460537, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07468391209840775 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8, + 4, + 0, + 12, + 21, + 25, + 1, + 19, + 0, + 4 + ], + "head_selection_time": "0:00:00.159105" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e90625c54 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2419657538064811, + "mae": 0.12487872689962387, + "r2": 0.6259568929672241, + "score": -0.2419657538064811 + }, + "val": { + "rmse": 0.23774960884328808, + "mae": 0.134882390499115, + "r2": 0.6224357485771179, + "score": -0.23774960884328808 + }, + "test": { + "rmse": 0.23837226942825876, + "mae": 0.1499200165271759, + "r2": 0.6117146015167236, + "score": -0.23837226942825876 + } + }, + "time": "0:01:31.301729", + "diversity": { + "train": { + "mean_centered_corr": -0.01885932870209217, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04668666422367096 + }, + "val": { + "mean_centered_corr": -0.01955920085310936, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06358597427606583 + }, + "test": { + "mean_centered_corr": -0.01645486429333687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07755117863416672 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015113" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0.toml new file mode 100644 index 000000000..da93a9d0c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..c44267e3c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2319513494822566, + "mae": 0.12131161987781525, + "r2": 0.6562777161598206, + "score": -0.2319513494822566 + }, + "val": { + "rmse": 0.23490809371963584, + "mae": 0.12586961686611176, + "r2": 0.6314069032669067, + "score": -0.23490809371963584 + }, + "test": { + "rmse": 0.23175764135614477, + "mae": 0.14321303367614746, + "r2": 0.6329647898674011, + "score": -0.23175764135614477 + } + }, + "time": "0:01:31.301729", + "diversity": { + "train": { + "mean_centered_corr": -0.01885932870209217, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04668666422367096 + }, + "val": { + "mean_centered_corr": -0.01955920085310936, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06358597427606583 + }, + "test": { + "mean_centered_corr": -0.01645486429333687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07755117863416672 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7c1e48457 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,701 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23423767040144092, + "mae": 0.12284845113754272, + "r2": 0.6494682431221008, + "score": -0.23423767040144092 + }, + "val": { + "rmse": 0.23309062467822936, + "mae": 0.12788444757461548, + "r2": 0.637088418006897, + "score": -0.23309062467822936 + }, + "test": { + "rmse": 0.2363639835864881, + "mae": 0.15300244092941284, + "r2": 0.6182296276092529, + "score": -0.2363639835864881 + } + }, + "time": "0:01:31.301729", + "diversity": { + "train": { + "mean_centered_corr": -0.01885932870209217, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04668666422367096 + }, + "val": { + "mean_centered_corr": -0.01955920085310936, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06358597427606583 + }, + "test": { + "mean_centered_corr": -0.01645486429333687, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07755117863416672 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 1, + 19, + 18, + 21, + 25, + 1 + ], + "head_selection_time": "0:00:00.128912" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..fc56d3b75 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23940312390912477, + "mae": 0.12330590933561325, + "r2": 0.6338378190994263, + "score": -0.23940312390912477 + }, + "val": { + "rmse": 0.2385343005293775, + "mae": 0.13736584782600403, + "r2": 0.6199393272399902, + "score": -0.2385343005293775 + }, + "test": { + "rmse": 0.24325157796763505, + "mae": 0.15736712515354156, + "r2": 0.5956560373306274, + "score": -0.24325157796763505 + } + }, + "time": "0:01:31.934812", + "diversity": { + "train": { + "mean_centered_corr": -0.019116338342428207, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047586847096681595 + }, + "val": { + "mean_centered_corr": -0.018940752372145653, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06937787681818008 + }, + "test": { + "mean_centered_corr": -0.01907981187105179, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07969599217176437 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015156" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0.toml new file mode 100644 index 000000000..1e0ac3792 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..fd189b307 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23138279613764395, + "mae": 0.12081756442785263, + "r2": 0.6579606533050537, + "score": -0.23138279613764395 + }, + "val": { + "rmse": 0.23465687967262416, + "mae": 0.1256990134716034, + "r2": 0.6321948170661926, + "score": -0.23465687967262416 + }, + "test": { + "rmse": 0.23108455049001858, + "mae": 0.14160698652267456, + "r2": 0.6350936889648438, + "score": -0.23108455049001858 + } + }, + "time": "0:01:31.934812", + "diversity": { + "train": { + "mean_centered_corr": -0.019116338342428207, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047586847096681595 + }, + "val": { + "mean_centered_corr": -0.018940752372145653, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06937787681818008 + }, + "test": { + "mean_centered_corr": -0.01907981187105179, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07969599217176437 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..06c18cf24 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,700 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23336989735039543, + "mae": 0.12107162177562714, + "r2": 0.6520606279373169, + "score": -0.23336989735039543 + }, + "val": { + "rmse": 0.23315395335227065, + "mae": 0.12548291683197021, + "r2": 0.6368911266326904, + "score": -0.23315395335227065 + }, + "test": { + "rmse": 0.2333720363863976, + "mae": 0.14587140083312988, + "r2": 0.6278335452079773, + "score": -0.2333720363863976 + } + }, + "time": "0:01:31.934812", + "diversity": { + "train": { + "mean_centered_corr": -0.019116338342428207, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047586847096681595 + }, + "val": { + "mean_centered_corr": -0.018940752372145653, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06937787681818008 + }, + "test": { + "mean_centered_corr": -0.01907981187105179, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07969599217176437 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 10, + 29, + 19, + 8, + 25, + 21 + ], + "head_selection_time": "0:00:00.115484" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..19c7c3dc8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2238305812388024, + "mae": 0.1228121742606163, + "r2": 0.6799242496490479, + "score": -0.2238305812388024 + }, + "val": { + "rmse": 0.23903748624498683, + "mae": 0.13215485215187073, + "r2": 0.618334174156189, + "score": -0.23903748624498683 + }, + "test": { + "rmse": 0.23879927467488832, + "mae": 0.14591430127620697, + "r2": 0.6103222370147705, + "score": -0.23879927467488832 + } + }, + "time": "0:01:42.466115", + "diversity": { + "train": { + "mean_centered_corr": -0.018108146265149117, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04599704593420029 + }, + "val": { + "mean_centered_corr": -0.017957618460059166, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06543198972940445 + }, + "test": { + "mean_centered_corr": -0.017061514779925346, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09027039259672165 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015148" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0.toml new file mode 100644 index 000000000..e3097adcc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..2bbcf6ab4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21851820244903958, + "mae": 0.11501690745353699, + "r2": 0.6949372887611389, + "score": -0.21851820244903958 + }, + "val": { + "rmse": 0.23496993373476788, + "mae": 0.12609615921974182, + "r2": 0.6312128305435181, + "score": -0.23496993373476788 + }, + "test": { + "rmse": 0.23346087685954336, + "mae": 0.14552420377731323, + "r2": 0.6275501251220703, + "score": -0.23346087685954336 + } + }, + "time": "0:01:42.466115", + "diversity": { + "train": { + "mean_centered_corr": -0.018108146265149117, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04599704593420029 + }, + "val": { + "mean_centered_corr": -0.017957618460059166, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06543198972940445 + }, + "test": { + "mean_centered_corr": -0.017061514779925346, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09027039259672165 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..bb5f598a0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,700 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22150166122823217, + "mae": 0.11824827641248703, + "r2": 0.6865503191947937, + "score": -0.22150166122823217 + }, + "val": { + "rmse": 0.23308791568735607, + "mae": 0.12669241428375244, + "r2": 0.6370968222618103, + "score": -0.23308791568735607 + }, + "test": { + "rmse": 0.23436618629413886, + "mae": 0.14811882376670837, + "r2": 0.6246559619903564, + "score": -0.23436618629413886 + } + }, + "time": "0:01:42.466115", + "diversity": { + "train": { + "mean_centered_corr": -0.018108146265149117, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04599704593420029 + }, + "val": { + "mean_centered_corr": -0.017957618460059166, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06543198972940445 + }, + "test": { + "mean_centered_corr": -0.017061514779925346, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09027039259672165 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 19, + 0, + 22, + 27, + 6, + 21, + 4 + ], + "head_selection_time": "0:00:00.115622" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..562d05f05 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2339843912395287, + "mae": 0.12764866650104523, + "r2": 0.6502258777618408, + "score": -0.2339843912395287 + }, + "val": { + "rmse": 0.23781616129168393, + "mae": 0.13649939000606537, + "r2": 0.6222243309020996, + "score": -0.23781616129168393 + }, + "test": { + "rmse": 0.24546581719417238, + "mae": 0.1635962575674057, + "r2": 0.5882613658905029, + "score": -0.24546581719417238 + } + }, + "time": "0:01:37.486772", + "diversity": { + "train": { + "mean_centered_corr": -0.018235279247164726, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0474671870470047 + }, + "val": { + "mean_centered_corr": -0.01877567730844021, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06657248735427856 + }, + "test": { + "mean_centered_corr": -0.016726242378354073, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08910585194826126 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29 + ], + "head_selection_time": "0:00:00.015175" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0.toml new file mode 100644 index 000000000..77cc09695 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..b34538bfa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22313156758053915, + "mae": 0.11779030412435532, + "r2": 0.6819202899932861, + "score": -0.22313156758053915 + }, + "val": { + "rmse": 0.23492098632735753, + "mae": 0.12972819805145264, + "r2": 0.6313664317131042, + "score": -0.23492098632735753 + }, + "test": { + "rmse": 0.2411043186306478, + "mae": 0.15983712673187256, + "r2": 0.6027631163597107, + "score": -0.2411043186306478 + } + }, + "time": "0:01:37.486772", + "diversity": { + "train": { + "mean_centered_corr": -0.018235279247164726, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0474671870470047 + }, + "val": { + "mean_centered_corr": -0.01877567730844021, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06657248735427856 + }, + "test": { + "mean_centered_corr": -0.016726242378354073, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08910585194826126 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..fa83f78cf --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,704 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22471723040163902, + "mae": 0.11835948377847672, + "r2": 0.6773834228515625, + "score": -0.22471723040163902 + }, + "val": { + "rmse": 0.23341984830196463, + "mae": 0.12756545841693878, + "r2": 0.636062502861023, + "score": -0.23341984830196463 + }, + "test": { + "rmse": 0.24027958886401746, + "mae": 0.15710820257663727, + "r2": 0.6054760813713074, + "score": -0.24027958886401746 + } + }, + "time": "0:01:37.486772", + "diversity": { + "train": { + "mean_centered_corr": -0.018235279247164726, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0474671870470047 + }, + "val": { + "mean_centered_corr": -0.01877567730844021, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06657248735427856 + }, + "test": { + "mean_centered_corr": -0.016726242378354073, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08910585194826126 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 29, + 24, + 4, + 19, + 25, + 7, + 10, + 27, + 22, + 6, + 21 + ], + "head_selection_time": "0:00:00.173692" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..28721b811 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23824613343670828, + "mae": 0.12383664399385452, + "r2": 0.6373684406280518, + "score": -0.23824613343670828 + }, + "val": { + "rmse": 0.23735456374234598, + "mae": 0.13839766383171082, + "r2": 0.6236894130706787, + "score": -0.23735456374234598 + }, + "test": { + "rmse": 0.24576564896936498, + "mae": 0.16598248481750488, + "r2": 0.5872548818588257, + "score": -0.24576564896936498 + } + }, + "time": "0:01:31.618555", + "diversity": { + "train": { + "mean_centered_corr": -0.018080253154039383, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0466311052441597 + }, + "val": { + "mean_centered_corr": -0.019496381282806396, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06081623584032059 + }, + "test": { + "mean_centered_corr": -0.019498975947499275, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08078814297914505 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015235" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0.toml new file mode 100644 index 000000000..0746ca938 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..b68def28e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23137620304543002, + "mae": 0.12124308198690414, + "r2": 0.65798020362854, + "score": -0.23137620304543002 + }, + "val": { + "rmse": 0.23479386888101392, + "mae": 0.12786690890789032, + "r2": 0.6317652463912964, + "score": -0.23479386888101392 + }, + "test": { + "rmse": 0.23419678588244902, + "mae": 0.1489223688840866, + "r2": 0.6251983642578125, + "score": -0.23419678588244902 + } + }, + "time": "0:01:31.618555", + "diversity": { + "train": { + "mean_centered_corr": -0.018080253154039383, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0466311052441597 + }, + "val": { + "mean_centered_corr": -0.019496381282806396, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06081623584032059 + }, + "test": { + "mean_centered_corr": -0.019498975947499275, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08078814297914505 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d21e1ac4f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2340414296102543, + "mae": 0.11944642663002014, + "r2": 0.6500552892684937, + "score": -0.2340414296102543 + }, + "val": { + "rmse": 0.23287923675610403, + "mae": 0.12622950971126556, + "r2": 0.6377463340759277, + "score": -0.23287923675610403 + }, + "test": { + "rmse": 0.23928534680182742, + "mae": 0.15746626257896423, + "r2": 0.6087342500686646, + "score": -0.23928534680182742 + } + }, + "time": "0:01:31.618555", + "diversity": { + "train": { + "mean_centered_corr": -0.018080253154039383, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0466311052441597 + }, + "val": { + "mean_centered_corr": -0.019496381282806396, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06081623584032059 + }, + "test": { + "mean_centered_corr": -0.019498975947499275, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08078814297914505 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 21, + 1, + 20, + 25, + 10 + ], + "head_selection_time": "0:00:00.101709" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..383df988d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2426666319444478, + "mae": 0.12620143592357635, + "r2": 0.6237868070602417, + "score": -0.2426666319444478 + }, + "val": { + "rmse": 0.2389538449467578, + "mae": 0.1335434764623642, + "r2": 0.6186012029647827, + "score": -0.2389538449467578 + }, + "test": { + "rmse": 0.24008690564440552, + "mae": 0.15347233414649963, + "r2": 0.606108546257019, + "score": -0.24008690564440552 + } + }, + "time": "0:01:32.555735", + "diversity": { + "train": { + "mean_centered_corr": -0.019036727026104927, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04819515720009804 + }, + "val": { + "mean_centered_corr": -0.019247591495513916, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0649481937289238 + }, + "test": { + "mean_centered_corr": -0.0215106550604105, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08662677556276321 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015189" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0.toml new file mode 100644 index 000000000..636067167 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..df6f95a5c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2317936605649109, + "mae": 0.12053651362657547, + "r2": 0.6567448973655701, + "score": -0.2317936605649109 + }, + "val": { + "rmse": 0.234848704054223, + "mae": 0.12717075645923615, + "r2": 0.6315932273864746, + "score": -0.234848704054223 + }, + "test": { + "rmse": 0.23316213384908455, + "mae": 0.14641331136226654, + "r2": 0.6285027265548706, + "score": -0.23316213384908455 + } + }, + "time": "0:01:32.555735", + "diversity": { + "train": { + "mean_centered_corr": -0.019036727026104927, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04819515720009804 + }, + "val": { + "mean_centered_corr": -0.019247591495513916, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0649481937289238 + }, + "test": { + "mean_centered_corr": -0.0215106550604105, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08662677556276321 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1475d6680 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,701 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2328109289922769, + "mae": 0.12201089411973953, + "r2": 0.6537253856658936, + "score": -0.2328109289922769 + }, + "val": { + "rmse": 0.2336424824585913, + "mae": 0.1279032826423645, + "r2": 0.6353679299354553, + "score": -0.2336424824585913 + }, + "test": { + "rmse": 0.23903173547895684, + "mae": 0.15682479739189148, + "r2": 0.6095632314682007, + "score": -0.23903173547895684 + } + }, + "time": "0:01:32.555735", + "diversity": { + "train": { + "mean_centered_corr": -0.019036727026104927, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04819515720009804 + }, + "val": { + "mean_centered_corr": -0.019247591495513916, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0649481937289238 + }, + "test": { + "mean_centered_corr": -0.0215106550604105, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08662677556276321 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 21, + 19, + 25, + 12, + 25, + 3, + 17 + ], + "head_selection_time": "0:00:00.130557" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..21317ca6a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2394503772198336, + "mae": 0.12332300841808319, + "r2": 0.6336932182312012, + "score": -0.2394503772198336 + }, + "val": { + "rmse": 0.23936406325960766, + "mae": 0.1317620724439621, + "r2": 0.6172905564308167, + "score": -0.23936406325960766 + }, + "test": { + "rmse": 0.23718997553210766, + "mae": 0.1439189463853836, + "r2": 0.6155567169189453, + "score": -0.23718997553210766 + } + }, + "time": "0:01:31.646908", + "diversity": { + "train": { + "mean_centered_corr": -0.01940109021961689, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047315195202827454 + }, + "val": { + "mean_centered_corr": -0.018565993756055832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06468198448419571 + }, + "test": { + "mean_centered_corr": -0.02023249678313732, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08129923790693283 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28 + ], + "head_selection_time": "0:00:00.018530" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0.toml new file mode 100644 index 000000000..e6f43bff5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..f5307f814 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23281691342209276, + "mae": 0.12255796045064926, + "r2": 0.6537076234817505, + "score": -0.23281691342209276 + }, + "val": { + "rmse": 0.23504563388193753, + "mae": 0.12878526747226715, + "r2": 0.6309751272201538, + "score": -0.23504563388193753 + }, + "test": { + "rmse": 0.2337165723322532, + "mae": 0.14694184064865112, + "r2": 0.6267338395118713, + "score": -0.2337165723322532 + } + }, + "time": "0:01:31.646908", + "diversity": { + "train": { + "mean_centered_corr": -0.01940109021961689, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047315195202827454 + }, + "val": { + "mean_centered_corr": -0.018565993756055832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06468198448419571 + }, + "test": { + "mean_centered_corr": -0.02023249678313732, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08129923790693283 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..eed37620b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,704 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2342020668363707, + "mae": 0.12203548103570938, + "r2": 0.6495747566223145, + "score": -0.2342020668363707 + }, + "val": { + "rmse": 0.23354782526985796, + "mae": 0.12771733105182648, + "r2": 0.6356633305549622, + "score": -0.23354782526985796 + }, + "test": { + "rmse": 0.23343612653897647, + "mae": 0.1471332311630249, + "r2": 0.6276291012763977, + "score": -0.23343612653897647 + } + }, + "time": "0:01:31.646908", + "diversity": { + "train": { + "mean_centered_corr": -0.01940109021961689, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047315195202827454 + }, + "val": { + "mean_centered_corr": -0.018565993756055832, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06468198448419571 + }, + "test": { + "mean_centered_corr": -0.02023249678313732, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08129923790693283 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28, + 25, + 29, + 19, + 3, + 25, + 4, + 12, + 25, + 27, + 1 + ], + "head_selection_time": "0:00:00.203719" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7e81581cd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24306374897467978, + "mae": 0.1263284981250763, + "r2": 0.6225544810295105, + "score": -0.24306374897467978 + }, + "val": { + "rmse": 0.238226555972743, + "mae": 0.13456811010837555, + "r2": 0.6209193468093872, + "score": -0.238226555972743 + }, + "test": { + "rmse": 0.2428780524434567, + "mae": 0.16094788908958435, + "r2": 0.5968968868255615, + "score": -0.2428780524434567 + } + }, + "time": "0:01:29.515084", + "diversity": { + "train": { + "mean_centered_corr": -0.019190935418009758, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04763631150126457 + }, + "val": { + "mean_centered_corr": -0.018849357962608337, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06253517419099808 + }, + "test": { + "mean_centered_corr": -0.020511005073785782, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08172544091939926 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015435" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0.toml new file mode 100644 index 000000000..e60ef2e8b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..13996eb16 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23280478439090932, + "mae": 0.12093566358089447, + "r2": 0.6537436842918396, + "score": -0.23280478439090932 + }, + "val": { + "rmse": 0.2351138707244049, + "mae": 0.12775011360645294, + "r2": 0.6307608485221863, + "score": -0.2351138707244049 + }, + "test": { + "rmse": 0.23346224116260836, + "mae": 0.14639027416706085, + "r2": 0.6275457739830017, + "score": -0.23346224116260836 + } + }, + "time": "0:01:29.515084", + "diversity": { + "train": { + "mean_centered_corr": -0.019190935418009758, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04763631150126457 + }, + "val": { + "mean_centered_corr": -0.018849357962608337, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06253517419099808 + }, + "test": { + "mean_centered_corr": -0.020511005073785782, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08172544091939926 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..009a3c336 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,702 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23307017461488594, + "mae": 0.12112323194742203, + "r2": 0.6529538035392761, + "score": -0.23307017461488594 + }, + "val": { + "rmse": 0.23345663231013183, + "mae": 0.1278393566608429, + "r2": 0.6359478235244751, + "score": -0.23345663231013183 + }, + "test": { + "rmse": 0.23687685165587416, + "mae": 0.1535518318414688, + "r2": 0.6165710687637329, + "score": -0.23687685165587416 + } + }, + "time": "0:01:29.515084", + "diversity": { + "train": { + "mean_centered_corr": -0.019190935418009758, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04763631150126457 + }, + "val": { + "mean_centered_corr": -0.018849357962608337, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06253517419099808 + }, + "test": { + "mean_centered_corr": -0.020511005073785782, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08172544091939926 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 21, + 25, + 28, + 19, + 3, + 25, + 12, + 25 + ], + "head_selection_time": "0:00:00.145632" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bbe8c1af4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2285154456765539, + "mae": 0.11967622488737106, + "r2": 0.6663854122161865, + "score": -0.2285154456765539 + }, + "val": { + "rmse": 0.23774841016444426, + "mae": 0.13070452213287354, + "r2": 0.6224395036697388, + "score": -0.23774841016444426 + }, + "test": { + "rmse": 0.24035398870465244, + "mae": 0.15418551862239838, + "r2": 0.605231761932373, + "score": -0.24035398870465244 + } + }, + "time": "0:01:36.865989", + "diversity": { + "train": { + "mean_centered_corr": -0.018838467076420784, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04630524292588234 + }, + "val": { + "mean_centered_corr": -0.01868019439280033, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0651327446103096 + }, + "test": { + "mean_centered_corr": -0.01751791313290596, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08838899433612823 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015153" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0.toml new file mode 100644 index 000000000..3499a8693 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..215a7ee15 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22344105424254884, + "mae": 0.11854980140924454, + "r2": 0.6810373663902283, + "score": -0.22344105424254884 + }, + "val": { + "rmse": 0.23512809084220013, + "mae": 0.12858927249908447, + "r2": 0.6307162046432495, + "score": -0.23512809084220013 + }, + "test": { + "rmse": 0.2355998696072441, + "mae": 0.15166118741035461, + "r2": 0.6206940412521362, + "score": -0.2355998696072441 + } + }, + "time": "0:01:36.865989", + "diversity": { + "train": { + "mean_centered_corr": -0.018838467076420784, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04630524292588234 + }, + "val": { + "mean_centered_corr": -0.01868019439280033, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0651327446103096 + }, + "test": { + "mean_centered_corr": -0.01751791313290596, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08838899433612823 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4b814cf15 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2240971488045897, + "mae": 0.11656086891889572, + "r2": 0.6791614294052124, + "score": -0.2240971488045897 + }, + "val": { + "rmse": 0.23360989779537025, + "mae": 0.12700146436691284, + "r2": 0.6354696154594421, + "score": -0.23360989779537025 + }, + "test": { + "rmse": 0.24433293207334258, + "mae": 0.16488319635391235, + "r2": 0.5920531153678894, + "score": -0.24433293207334258 + } + }, + "time": "0:01:36.865989", + "diversity": { + "train": { + "mean_centered_corr": -0.018838467076420784, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04630524292588234 + }, + "val": { + "mean_centered_corr": -0.01868019439280033, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0651327446103096 + }, + "test": { + "mean_centered_corr": -0.01751791313290596, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08838899433612823 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 18, + 21, + 25, + 8 + ], + "head_selection_time": "0:00:00.086285" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8c74b4063 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24133726047749576, + "mae": 0.12366868555545807, + "r2": 0.6278974413871765, + "score": -0.24133726047749576 + }, + "val": { + "rmse": 0.23613982935624828, + "mae": 0.1324111670255661, + "r2": 0.6275312900543213, + "score": -0.23613982935624828 + }, + "test": { + "rmse": 0.23869431760070778, + "mae": 0.15286467969417572, + "r2": 0.6106647253036499, + "score": -0.23869431760070778 + } + }, + "time": "0:01:30.265637", + "diversity": { + "train": { + "mean_centered_corr": -0.019571561366319656, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04648323729634285 + }, + "val": { + "mean_centered_corr": -0.01869725063443184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06371068954467773 + }, + "test": { + "mean_centered_corr": -0.018871210515499115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07722875475883484 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015136" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0.toml new file mode 100644 index 000000000..bf36690ed --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..35d492921 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23293543424527074, + "mae": 0.12151619791984558, + "r2": 0.6533549427986145, + "score": -0.23293543424527074 + }, + "val": { + "rmse": 0.23518098688395506, + "mae": 0.12663617730140686, + "r2": 0.6305500268936157, + "score": -0.23518098688395506 + }, + "test": { + "rmse": 0.23184570253369863, + "mae": 0.14382076263427734, + "r2": 0.6326858401298523, + "score": -0.23184570253369863 + } + }, + "time": "0:01:30.265637", + "diversity": { + "train": { + "mean_centered_corr": -0.019571561366319656, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04648323729634285 + }, + "val": { + "mean_centered_corr": -0.01869725063443184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06371068954467773 + }, + "test": { + "mean_centered_corr": -0.018871210515499115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07722875475883484 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7d7e41dcc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2358128076066539, + "mae": 0.1227794662117958, + "r2": 0.6447380781173706, + "score": -0.2358128076066539 + }, + "val": { + "rmse": 0.23307964467970407, + "mae": 0.12707222998142242, + "r2": 0.6371225714683533, + "score": -0.23307964467970407 + }, + "test": { + "rmse": 0.23920139503937368, + "mae": 0.1561342030763626, + "r2": 0.6090087890625, + "score": -0.23920139503937368 + } + }, + "time": "0:01:30.265637", + "diversity": { + "train": { + "mean_centered_corr": -0.019571561366319656, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04648323729634285 + }, + "val": { + "mean_centered_corr": -0.01869725063443184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06371068954467773 + }, + "test": { + "mean_centered_corr": -0.018871210515499115, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07722875475883484 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 8, + 25, + 21 + ], + "head_selection_time": "0:00:00.087431" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ea48767f9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2313955309804549, + "mae": 0.12607093155384064, + "r2": 0.6579230427742004, + "score": -0.2313955309804549 + }, + "val": { + "rmse": 0.23924971586268343, + "mae": 0.13559652864933014, + "r2": 0.6176561117172241, + "score": -0.23924971586268343 + }, + "test": { + "rmse": 0.23943478014622727, + "mae": 0.1478048413991928, + "r2": 0.6082454323768616, + "score": -0.23943478014622727 + } + }, + "time": "0:01:40.717588", + "diversity": { + "train": { + "mean_centered_corr": -0.018105898052453995, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04715852439403534 + }, + "val": { + "mean_centered_corr": -0.019619733095169067, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06463336944580078 + }, + "test": { + "mean_centered_corr": -0.017858682200312614, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08570578694343567 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.015407" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0.toml new file mode 100644 index 000000000..0f3309d70 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..62dc1ebc2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21986647495899417, + "mae": 0.11594657599925995, + "r2": 0.6911611557006836, + "score": -0.21986647495899417 + }, + "val": { + "rmse": 0.23473991754724038, + "mae": 0.12713272869586945, + "r2": 0.6319344639778137, + "score": -0.23473991754724038 + }, + "test": { + "rmse": 0.23324559206850415, + "mae": 0.14693446457386017, + "r2": 0.628236711025238, + "score": -0.23324559206850415 + } + }, + "time": "0:01:40.717588", + "diversity": { + "train": { + "mean_centered_corr": -0.018105898052453995, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04715852439403534 + }, + "val": { + "mean_centered_corr": -0.019619733095169067, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06463336944580078 + }, + "test": { + "mean_centered_corr": -0.017858682200312614, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08570578694343567 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..d633c837c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,707 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21944796383995846, + "mae": 0.11816469579935074, + "r2": 0.6923357844352722, + "score": -0.21944796383995846 + }, + "val": { + "rmse": 0.233123281922542, + "mae": 0.127369225025177, + "r2": 0.6369866728782654, + "score": -0.233123281922542 + }, + "test": { + "rmse": 0.2324621021496061, + "mae": 0.14417536556720734, + "r2": 0.6307300925254822, + "score": -0.2324621021496061 + } + }, + "time": "0:01:40.717588", + "diversity": { + "train": { + "mean_centered_corr": -0.018105898052453995, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04715852439403534 + }, + "val": { + "mean_centered_corr": -0.019619733095169067, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06463336944580078 + }, + "test": { + "mean_centered_corr": -0.017858682200312614, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08570578694343567 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 8, + 21, + 31, + 0, + 22, + 14, + 10, + 21, + 31, + 19, + 20, + 4, + 21, + 0 + ], + "head_selection_time": "0:00:00.225657" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5465e4d4c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2393712066992257, + "mae": 0.12284750491380692, + "r2": 0.6339354515075684, + "score": -0.2393712066992257 + }, + "val": { + "rmse": 0.23604388522704514, + "mae": 0.1302504986524582, + "r2": 0.6278339624404907, + "score": -0.23604388522704514 + }, + "test": { + "rmse": 0.2413116661749965, + "mae": 0.15745559334754944, + "r2": 0.6020795702934265, + "score": -0.2413116661749965 + } + }, + "time": "0:01:31.102179", + "diversity": { + "train": { + "mean_centered_corr": -0.019335079938173294, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045941311866045 + }, + "val": { + "mean_centered_corr": -0.019422883167862892, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05757637694478035 + }, + "test": { + "mean_centered_corr": -0.018836498260498047, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07515785843133926 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015295" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0.toml new file mode 100644 index 000000000..49391d2ba --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..a1d1c3a33 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23172434955038107, + "mae": 0.11937171965837479, + "r2": 0.6569501161575317, + "score": -0.23172434955038107 + }, + "val": { + "rmse": 0.23468678716863678, + "mae": 0.12593677639961243, + "r2": 0.6321010589599609, + "score": -0.23468678716863678 + }, + "test": { + "rmse": 0.23217868905016578, + "mae": 0.14450614154338837, + "r2": 0.6316299438476562, + "score": -0.23217868905016578 + } + }, + "time": "0:01:31.102179", + "diversity": { + "train": { + "mean_centered_corr": -0.019335079938173294, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045941311866045 + }, + "val": { + "mean_centered_corr": -0.019422883167862892, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05757637694478035 + }, + "test": { + "mean_centered_corr": -0.018836498260498047, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07515785843133926 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ede0cf086 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,701 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2339039041660179, + "mae": 0.12130481749773026, + "r2": 0.6504664421081543, + "score": -0.2339039041660179 + }, + "val": { + "rmse": 0.23298097729336806, + "mae": 0.12618261575698853, + "r2": 0.6374297142028809, + "score": -0.23298097729336806 + }, + "test": { + "rmse": 0.23673558379896892, + "mae": 0.15379810333251953, + "r2": 0.6170282959938049, + "score": -0.23673558379896892 + } + }, + "time": "0:01:31.102179", + "diversity": { + "train": { + "mean_centered_corr": -0.019335079938173294, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045941311866045 + }, + "val": { + "mean_centered_corr": -0.019422883167862892, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05757637694478035 + }, + "test": { + "mean_centered_corr": -0.018836498260498047, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07515785843133926 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 1, + 16, + 0, + 21, + 25, + 20, + 1 + ], + "head_selection_time": "0:00:00.129256" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ee07af13d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2402521219909073, + "mae": 0.12393063306808472, + "r2": 0.6312361359596252, + "score": -0.2402521219909073 + }, + "val": { + "rmse": 0.2369960693297996, + "mae": 0.1321815848350525, + "r2": 0.6248252987861633, + "score": -0.2369960693297996 + }, + "test": { + "rmse": 0.2449247661219165, + "mae": 0.16224433481693268, + "r2": 0.5900744199752808, + "score": -0.2449247661219165 + } + }, + "time": "0:01:31.861899", + "diversity": { + "train": { + "mean_centered_corr": -0.01910976506769657, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04582935944199562 + }, + "val": { + "mean_centered_corr": -0.01908961497247219, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06326822191476822 + }, + "test": { + "mean_centered_corr": -0.02014167793095112, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08288853615522385 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015320" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0.toml new file mode 100644 index 000000000..968761935 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..fe64ac728 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23193927958696722, + "mae": 0.12050655484199524, + "r2": 0.6563134789466858, + "score": -0.23193927958696722 + }, + "val": { + "rmse": 0.23531282668072034, + "mae": 0.1256631463766098, + "r2": 0.6301356554031372, + "score": -0.23531282668072034 + }, + "test": { + "rmse": 0.2329136030473919, + "mae": 0.1449238359928131, + "r2": 0.6292942762374878, + "score": -0.2329136030473919 + } + }, + "time": "0:01:31.861899", + "diversity": { + "train": { + "mean_centered_corr": -0.01910976506769657, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04582935944199562 + }, + "val": { + "mean_centered_corr": -0.01908961497247219, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06326822191476822 + }, + "test": { + "mean_centered_corr": -0.02014167793095112, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08288853615522385 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..13bb585de --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23367056683958856, + "mae": 0.12022486329078674, + "r2": 0.6511635184288025, + "score": -0.23367056683958856 + }, + "val": { + "rmse": 0.23320698567170814, + "mae": 0.12736979126930237, + "r2": 0.6367259621620178, + "score": -0.23320698567170814 + }, + "test": { + "rmse": 0.24367480279619608, + "mae": 0.16372117400169373, + "r2": 0.5942478179931641, + "score": -0.24367480279619608 + } + }, + "time": "0:01:31.861899", + "diversity": { + "train": { + "mean_centered_corr": -0.01910976506769657, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04582935944199562 + }, + "val": { + "mean_centered_corr": -0.01908961497247219, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06326822191476822 + }, + "test": { + "mean_centered_corr": -0.02014167793095112, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08288853615522385 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 21, + 18, + 25, + 10 + ], + "head_selection_time": "0:00:00.101265" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..e5e21eca5 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23893874555528044, + "mae": 0.13176758587360382, + "r2": 0.6352569460868835, + "score": -0.23893874555528044 + }, + "val": { + "rmse": 0.23723977382385666, + "mae": 0.13325777649879456, + "r2": 0.6240533590316772, + "score": -0.23723977382385666 + }, + "test": { + "rmse": 0.24324646285162105, + "mae": 0.15948642790317535, + "r2": 0.5956730842590332, + "score": -0.24324646285162105 + } + }, + "time": "0:01:30.961009", + "diversity": { + "train": { + "mean_centered_corr": -0.0197697002440691, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046830255538225174 + }, + "val": { + "mean_centered_corr": -0.01885363645851612, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06196551397442818 + }, + "test": { + "mean_centered_corr": -0.01950950175523758, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07843607664108276 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015095" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0.toml new file mode 100644 index 000000000..c819006f0 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..4f6082808 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23177381131405664, + "mae": 0.11973857134580612, + "r2": 0.6568037271499634, + "score": -0.23177381131405664 + }, + "val": { + "rmse": 0.23489533520890316, + "mae": 0.12537376582622528, + "r2": 0.6314469575881958, + "score": -0.23489533520890316 + }, + "test": { + "rmse": 0.23252367146541786, + "mae": 0.14447112381458282, + "r2": 0.6305344700813293, + "score": -0.23252367146541786 + } + }, + "time": "0:01:30.961009", + "diversity": { + "train": { + "mean_centered_corr": -0.0197697002440691, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046830255538225174 + }, + "val": { + "mean_centered_corr": -0.01885363645851612, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06196551397442818 + }, + "test": { + "mean_centered_corr": -0.01950950175523758, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07843607664108276 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c181f9775 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23515993444908453, + "mae": 0.12149348109960556, + "r2": 0.6467025279998779, + "score": -0.23515993444908453 + }, + "val": { + "rmse": 0.23277517922062366, + "mae": 0.1261647343635559, + "r2": 0.6380699872970581, + "score": -0.23277517922062366 + }, + "test": { + "rmse": 0.239397685367944, + "mae": 0.1571250855922699, + "r2": 0.608366847038269, + "score": -0.239397685367944 + } + }, + "time": "0:01:30.961009", + "diversity": { + "train": { + "mean_centered_corr": -0.0197697002440691, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046830255538225174 + }, + "val": { + "mean_centered_corr": -0.01885363645851612, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06196551397442818 + }, + "test": { + "mean_centered_corr": -0.01950950175523758, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07843607664108276 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 0, + 4, + 21, + 25, + 1 + ], + "head_selection_time": "0:00:00.086399" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6322e1094 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.23146582564861154, + "mae": 0.12501563131809235, + "r2": 0.6577152013778687, + "score": -0.23146582564861154 + }, + "val": { + "rmse": 0.23843676541966466, + "mae": 0.14031149446964264, + "r2": 0.6202501058578491, + "score": -0.23843676541966466 + }, + "test": { + "rmse": 0.2470445431022663, + "mae": 0.16841350495815277, + "r2": 0.5829480886459351, + "score": -0.2470445431022663 + } + }, + "time": "0:01:44.046551", + "diversity": { + "train": { + "mean_centered_corr": -0.01830325461924076, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04772296920418739 + }, + "val": { + "mean_centered_corr": -0.019104016944766045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06741341203451157 + }, + "test": { + "mean_centered_corr": -0.017741652205586433, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09074985980987549 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015187" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0.toml new file mode 100644 index 000000000..86978b7b8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..511d952ad --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2188970803621353, + "mae": 0.11487266421318054, + "r2": 0.6938785314559937, + "score": -0.2188970803621353 + }, + "val": { + "rmse": 0.23493513883666117, + "mae": 0.12670956552028656, + "r2": 0.6313220262527466, + "score": -0.23493513883666117 + }, + "test": { + "rmse": 0.2324465730511325, + "mae": 0.14363469183444977, + "r2": 0.6307794451713562, + "score": -0.2324465730511325 + } + }, + "time": "0:01:44.046551", + "diversity": { + "train": { + "mean_centered_corr": -0.01830325461924076, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04772296920418739 + }, + "val": { + "mean_centered_corr": -0.019104016944766045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06741341203451157 + }, + "test": { + "mean_centered_corr": -0.017741652205586433, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09074985980987549 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2b2b0f319 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,700 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2202455368136436, + "mae": 0.11561281979084015, + "r2": 0.6900953054428101, + "score": -0.2202455368136436 + }, + "val": { + "rmse": 0.23360278549070665, + "mae": 0.12652066349983215, + "r2": 0.6354918479919434, + "score": -0.23360278549070665 + }, + "test": { + "rmse": 0.23177048418825977, + "mae": 0.1428041160106659, + "r2": 0.6329240798950195, + "score": -0.23177048418825977 + } + }, + "time": "0:01:44.046551", + "diversity": { + "train": { + "mean_centered_corr": -0.01830325461924076, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04772296920418739 + }, + "val": { + "mean_centered_corr": -0.019104016944766045, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06741341203451157 + }, + "test": { + "mean_centered_corr": -0.017741652205586433, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09074985980987549 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 25, + 19, + 0, + 3, + 21, + 1, + 20 + ], + "head_selection_time": "0:00:00.116231" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..7777a7a3e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24224300671355675, + "mae": 0.12633994221687317, + "r2": 0.6250991821289062, + "score": -0.24224300671355675 + }, + "val": { + "rmse": 0.23790953518728727, + "mae": 0.1355382353067398, + "r2": 0.6219276189804077, + "score": -0.23790953518728727 + }, + "test": { + "rmse": 0.23821708722105353, + "mae": 0.15011507272720337, + "r2": 0.6122199892997742, + "score": -0.23821708722105353 + } + }, + "time": "0:01:29.035550", + "diversity": { + "train": { + "mean_centered_corr": -0.0192310381680727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04764547199010849 + }, + "val": { + "mean_centered_corr": -0.018086779862642288, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0648212805390358 + }, + "test": { + "mean_centered_corr": -0.01891481876373291, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08217111229896545 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015344" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0.toml new file mode 100644 index 000000000..ddec6331a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..f9245b832 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23216875702743484, + "mae": 0.12165454775094986, + "r2": 0.6556330919265747, + "score": -0.23216875702743484 + }, + "val": { + "rmse": 0.2350836134726586, + "mae": 0.128606379032135, + "r2": 0.6308558583259583, + "score": -0.2350836134726586 + }, + "test": { + "rmse": 0.23409832677668932, + "mae": 0.1483079195022583, + "r2": 0.6255134344100952, + "score": -0.23409832677668932 + } + }, + "time": "0:01:29.035550", + "diversity": { + "train": { + "mean_centered_corr": -0.0192310381680727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04764547199010849 + }, + "val": { + "mean_centered_corr": -0.018086779862642288, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0648212805390358 + }, + "test": { + "mean_centered_corr": -0.01891481876373291, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08217111229896545 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1c6bc379c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,703 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23386279403869487, + "mae": 0.1229424923658371, + "r2": 0.6505893468856812, + "score": -0.23386279403869487 + }, + "val": { + "rmse": 0.23395800051802765, + "mae": 0.1274479776620865, + "r2": 0.634382426738739, + "score": -0.23395800051802765 + }, + "test": { + "rmse": 0.23212124110153423, + "mae": 0.1446542590856552, + "r2": 0.6318122148513794, + "score": -0.23212124110153423 + } + }, + "time": "0:01:29.035550", + "diversity": { + "train": { + "mean_centered_corr": -0.0192310381680727, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04764547199010849 + }, + "val": { + "mean_centered_corr": -0.018086779862642288, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0648212805390358 + }, + "test": { + "mean_centered_corr": -0.01891481876373291, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08217111229896545 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 28, + 19, + 21, + 0, + 24, + 29, + 10, + 16, + 3 + ], + "head_selection_time": "0:00:00.157884" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..db37da897 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23859920550431532, + "mae": 0.12155521661043167, + "r2": 0.6362928152084351, + "score": -0.23859920550431532 + }, + "val": { + "rmse": 0.23752709943108333, + "mae": 0.1321943998336792, + "r2": 0.6231421232223511, + "score": -0.23752709943108333 + }, + "test": { + "rmse": 0.23270363118336854, + "mae": 0.14155620336532593, + "r2": 0.6299623250961304, + "score": -0.23270363118336854 + } + }, + "time": "0:01:30.732738", + "diversity": { + "train": { + "mean_centered_corr": -0.018462583422660828, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047156888991594315 + }, + "val": { + "mean_centered_corr": -0.017515555024147034, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06680545955896378 + }, + "test": { + "mean_centered_corr": -0.017912454903125763, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08376385271549225 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28 + ], + "head_selection_time": "0:00:00.015330" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0.toml new file mode 100644 index 000000000..0dfe890fe --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 2 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..5e260f037 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23201801570565175, + "mae": 0.1213398203253746, + "r2": 0.6560801267623901, + "score": -0.23201801570565175 + }, + "val": { + "rmse": 0.23493006464565203, + "mae": 0.1288696676492691, + "r2": 0.6313378810882568, + "score": -0.23493006464565203 + }, + "test": { + "rmse": 0.23367776476223437, + "mae": 0.14683480560779572, + "r2": 0.6268577575683594, + "score": -0.23367776476223437 + } + }, + "time": "0:01:30.732738", + "diversity": { + "train": { + "mean_centered_corr": -0.018462583422660828, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047156888991594315 + }, + "val": { + "mean_centered_corr": -0.017515555024147034, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06680545955896378 + }, + "test": { + "mean_centered_corr": -0.017912454903125763, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08376385271549225 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..4e3ae064e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,705 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 2, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4457512, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 2 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.232872877996383, + "mae": 0.12275310605764389, + "r2": 0.653541088104248, + "score": -0.232872877996383 + }, + "val": { + "rmse": 0.23302229095267976, + "mae": 0.12814900279045105, + "r2": 0.6373011469841003, + "score": -0.23302229095267976 + }, + "test": { + "rmse": 0.2330139696533979, + "mae": 0.146689772605896, + "r2": 0.6289746761322021, + "score": -0.2330139696533979 + } + }, + "time": "0:01:30.732738", + "diversity": { + "train": { + "mean_centered_corr": -0.018462583422660828, + "mean_pairwise_disagreement": NaN, + "member_std": 0.047156888991594315 + }, + "val": { + "mean_centered_corr": -0.017515555024147034, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06680545955896378 + }, + "test": { + "mean_centered_corr": -0.017912454903125763, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08376385271549225 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 28, + 19, + 25, + 4, + 12, + 24, + 0, + 21, + 25, + 29, + 19, + 4 + ], + "head_selection_time": "0:00:00.191270" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f019e5b6f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.2252595034159999, + "mae": 0.12715505063533783, + "r2": 0.6758245229721069, + "score": -0.2252595034159999 + }, + "val": { + "rmse": 0.23834036265829958, + "mae": 0.133585587143898, + "r2": 0.6205570697784424, + "score": -0.23834036265829958 + }, + "test": { + "rmse": 0.2355387485424496, + "mae": 0.14495041966438293, + "r2": 0.6208908557891846, + "score": -0.2355387485424496 + } + }, + "time": "0:02:14.105851", + "diversity": { + "train": { + "mean_centered_corr": -0.018248364329338074, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045475468039512634 + }, + "val": { + "mean_centered_corr": -0.01828787662088871, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06622428447008133 + }, + "test": { + "mean_centered_corr": -0.01435924507677555, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09353811293840408 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 19 + ], + "head_selection_time": "0:00:00.015604" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0.toml new file mode 100644 index 000000000..b350c1c90 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..d66ed52f1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22275971635451677, + "mae": 0.11979660391807556, + "r2": 0.6829795837402344, + "score": -0.22275971635451677 + }, + "val": { + "rmse": 0.23522175576712184, + "mae": 0.1303437203168869, + "r2": 0.6304218769073486, + "score": -0.23522175576712184 + }, + "test": { + "rmse": 0.2392933877459825, + "mae": 0.1575920134782791, + "r2": 0.6087079644203186, + "score": -0.2392933877459825 + } + }, + "time": "0:02:14.105851", + "diversity": { + "train": { + "mean_centered_corr": -0.018248364329338074, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045475468039512634 + }, + "val": { + "mean_centered_corr": -0.01828787662088871, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06622428447008133 + }, + "test": { + "mean_centered_corr": -0.01435924507677555, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09353811293840408 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7bbd01f02 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,701 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 814, + "metrics": { + "train": { + "rmse": 0.22522487065919897, + "mae": 0.1186138316988945, + "r2": 0.6759241819381714, + "score": -0.22522487065919897 + }, + "val": { + "rmse": 0.2328865391175455, + "mae": 0.12701237201690674, + "r2": 0.6377235651016235, + "score": -0.2328865391175455 + }, + "test": { + "rmse": 0.23281656140106766, + "mae": 0.14357610046863556, + "r2": 0.6296030879020691, + "score": -0.23281656140106766 + } + }, + "time": "0:02:14.105851", + "diversity": { + "train": { + "mean_centered_corr": -0.018248364329338074, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045475468039512634 + }, + "val": { + "mean_centered_corr": -0.01828787662088871, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06622428447008133 + }, + "test": { + "mean_centered_corr": -0.01435924507677555, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09353811293840408 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 19, + 8, + 4, + 29, + 7, + 8, + 1, + 21 + ], + "head_selection_time": "0:00:00.131174" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4d9024b29 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.229118968956047, + "mae": 0.12437356263399124, + "r2": 0.6646209359169006, + "score": -0.229118968956047 + }, + "val": { + "rmse": 0.23782636654662812, + "mae": 0.12977644801139832, + "r2": 0.6221919059753418, + "score": -0.23782636654662812 + }, + "test": { + "rmse": 0.23905601555002126, + "mae": 0.14474956691265106, + "r2": 0.6094838976860046, + "score": -0.23905601555002126 + } + }, + "time": "0:02:16.774478", + "diversity": { + "train": { + "mean_centered_corr": -0.018586045131087303, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045367300510406494 + }, + "val": { + "mean_centered_corr": -0.019293533638119698, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07069077342748642 + }, + "test": { + "mean_centered_corr": -0.01720191352069378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09131180495023727 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.015326" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0.toml new file mode 100644 index 000000000..8fc7a8082 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..d5fd99862 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21912932715302513, + "mae": 0.11480405181646347, + "r2": 0.6932286024093628, + "score": -0.21912932715302513 + }, + "val": { + "rmse": 0.2351262925795415, + "mae": 0.12626133859157562, + "r2": 0.6307218074798584, + "score": -0.2351262925795415 + }, + "test": { + "rmse": 0.23336620188006213, + "mae": 0.14624115824699402, + "r2": 0.627852201461792, + "score": -0.23336620188006213 + } + }, + "time": "0:02:16.774478", + "diversity": { + "train": { + "mean_centered_corr": -0.018586045131087303, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045367300510406494 + }, + "val": { + "mean_centered_corr": -0.019293533638119698, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07069077342748642 + }, + "test": { + "mean_centered_corr": -0.01720191352069378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09131180495023727 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..dff01cfa8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,700 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2210335747342936, + "mae": 0.11760202050209045, + "r2": 0.6878736615180969, + "score": -0.2210335747342936 + }, + "val": { + "rmse": 0.2332281025812143, + "mae": 0.12575659155845642, + "r2": 0.6366601586341858, + "score": -0.2332281025812143 + }, + "test": { + "rmse": 0.23533248824072742, + "mae": 0.14844603836536407, + "r2": 0.6215544939041138, + "score": -0.23533248824072742 + } + }, + "time": "0:02:16.774478", + "diversity": { + "train": { + "mean_centered_corr": -0.018586045131087303, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045367300510406494 + }, + "val": { + "mean_centered_corr": -0.019293533638119698, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07069077342748642 + }, + "test": { + "mean_centered_corr": -0.01720191352069378, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09131180495023727 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 8, + 27, + 19, + 0, + 25, + 21, + 4 + ], + "head_selection_time": "0:00:00.117123" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..50c13353b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2242175374679477, + "mae": 0.12167605012655258, + "r2": 0.6788166761398315, + "score": -0.2242175374679477 + }, + "val": { + "rmse": 0.2383424727161954, + "mae": 0.1329953521490097, + "r2": 0.6205503940582275, + "score": -0.2383424727161954 + }, + "test": { + "rmse": 0.23929541934545473, + "mae": 0.1496122032403946, + "r2": 0.6087013483047485, + "score": -0.23929541934545473 + } + }, + "time": "0:02:17.580078", + "diversity": { + "train": { + "mean_centered_corr": -0.018310561776161194, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04597850888967514 + }, + "val": { + "mean_centered_corr": -0.01918131671845913, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06879474967718124 + }, + "test": { + "mean_centered_corr": -0.016168270260095596, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08937344700098038 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015136" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0.toml new file mode 100644 index 000000000..6b87f0411 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..88bb0714f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21875939178741377, + "mae": 0.11528072506189346, + "r2": 0.6942634582519531, + "score": -0.21875939178741377 + }, + "val": { + "rmse": 0.23509613200143026, + "mae": 0.12603282928466797, + "r2": 0.6308165788650513, + "score": -0.23509613200143026 + }, + "test": { + "rmse": 0.23336054282826887, + "mae": 0.14659926295280457, + "r2": 0.6278702020645142, + "score": -0.23336054282826887 + } + }, + "time": "0:02:17.580078", + "diversity": { + "train": { + "mean_centered_corr": -0.018310561776161194, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04597850888967514 + }, + "val": { + "mean_centered_corr": -0.01918131671845913, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06879474967718124 + }, + "test": { + "mean_centered_corr": -0.016168270260095596, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08937344700098038 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..5d88fb9cb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p95_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21918426561919716, + "mae": 0.11884808540344238, + "r2": 0.6930747032165527, + "score": -0.21918426561919716 + }, + "val": { + "rmse": 0.23335380606300898, + "mae": 0.12806729972362518, + "r2": 0.6362683773040771, + "score": -0.23335380606300898 + }, + "test": { + "rmse": 0.23440792329336166, + "mae": 0.1488981395959854, + "r2": 0.62452232837677, + "score": -0.23440792329336166 + } + }, + "time": "0:02:17.580078", + "diversity": { + "train": { + "mean_centered_corr": -0.018310561776161194, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04597850888967514 + }, + "val": { + "mean_centered_corr": -0.01918131671845913, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06879474967718124 + }, + "test": { + "mean_centered_corr": -0.016168270260095596, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08937344700098038 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 25, + 19, + 21, + 8, + 31 + ], + "head_selection_time": "0:00:00.101201" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..c5a66d7c9 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2362965728252335, + "mae": 0.12153194099664688, + "r2": 0.643278956413269, + "score": -0.2362965728252335 + }, + "val": { + "rmse": 0.2357278561917395, + "mae": 0.1297774612903595, + "r2": 0.628829836845398, + "score": -0.2357278561917395 + }, + "test": { + "rmse": 0.24177902137278479, + "mae": 0.15532447397708893, + "r2": 0.6005367636680603, + "score": -0.24177902137278479 + } + }, + "time": "0:02:03.045766", + "diversity": { + "train": { + "mean_centered_corr": -0.018870852887630463, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044821128249168396 + }, + "val": { + "mean_centered_corr": -0.018474284559488297, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057242587208747864 + }, + "test": { + "mean_centered_corr": -0.01819882169365883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08083672821521759 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015341" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0.toml new file mode 100644 index 000000000..2fdcae837 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..c1382027a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.22965888604190204, + "mae": 0.12082202732563019, + "r2": 0.6630383729934692, + "score": -0.22965888604190204 + }, + "val": { + "rmse": 0.2346724530004704, + "mae": 0.12799137830734253, + "r2": 0.6321460008621216, + "score": -0.2346724530004704 + }, + "test": { + "rmse": 0.23472572946107031, + "mae": 0.15012341737747192, + "r2": 0.6235034465789795, + "score": -0.23472572946107031 + } + }, + "time": "0:02:03.045766", + "diversity": { + "train": { + "mean_centered_corr": -0.018870852887630463, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044821128249168396 + }, + "val": { + "mean_centered_corr": -0.018474284559488297, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057242587208747864 + }, + "test": { + "mean_centered_corr": -0.01819882169365883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08083672821521759 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ec97c5157 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23456546673625236, + "mae": 0.12042497098445892, + "r2": 0.6484864950180054, + "score": -0.23456546673625236 + }, + "val": { + "rmse": 0.2330191974784263, + "mae": 0.12631137669086456, + "r2": 0.6373107433319092, + "score": -0.2330191974784263 + }, + "test": { + "rmse": 0.24024111265333, + "mae": 0.15944169461727142, + "r2": 0.6056023836135864, + "score": -0.24024111265333 + } + }, + "time": "0:02:03.045766", + "diversity": { + "train": { + "mean_centered_corr": -0.018870852887630463, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044821128249168396 + }, + "val": { + "mean_centered_corr": -0.018474284559488297, + "mean_pairwise_disagreement": NaN, + "member_std": 0.057242587208747864 + }, + "test": { + "mean_centered_corr": -0.01819882169365883, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08083672821521759 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 21, + 0, + 10, + 1, + 28 + ], + "head_selection_time": "0:00:00.101866" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..485c6f813 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23715487802399932, + "mae": 0.12212099879980087, + "r2": 0.6406828165054321, + "score": -0.23715487802399932 + }, + "val": { + "rmse": 0.2361938000242818, + "mae": 0.130824014544487, + "r2": 0.6273610591888428, + "score": -0.2361938000242818 + }, + "test": { + "rmse": 0.23968299921179284, + "mae": 0.15190771222114563, + "r2": 0.6074327826499939, + "score": -0.23968299921179284 + } + }, + "time": "0:02:03.488922", + "diversity": { + "train": { + "mean_centered_corr": -0.019381657242774963, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04552893713116646 + }, + "val": { + "mean_centered_corr": -0.01903798058629036, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06320050358772278 + }, + "test": { + "mean_centered_corr": -0.018483227118849754, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08418284356594086 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015490" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0.toml new file mode 100644 index 000000000..705351034 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..a4ae477f4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23023425311605375, + "mae": 0.11950831860303879, + "r2": 0.6613479256629944, + "score": -0.23023425311605375 + }, + "val": { + "rmse": 0.23514965307924723, + "mae": 0.12576493620872498, + "r2": 0.6306484341621399, + "score": -0.23514965307924723 + }, + "test": { + "rmse": 0.23159657135010447, + "mae": 0.1426416039466858, + "r2": 0.6334748268127441, + "score": -0.23159657135010447 + } + }, + "time": "0:02:03.488922", + "diversity": { + "train": { + "mean_centered_corr": -0.019381657242774963, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04552893713116646 + }, + "val": { + "mean_centered_corr": -0.01903798058629036, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06320050358772278 + }, + "test": { + "mean_centered_corr": -0.018483227118849754, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08418284356594086 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..ccb8e9619 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,697 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23299274539756, + "mae": 0.12110579758882523, + "r2": 0.6531843543052673, + "score": -0.23299274539756 + }, + "val": { + "rmse": 0.23281693742350693, + "mae": 0.1263430267572403, + "r2": 0.6379401087760925, + "score": -0.23281693742350693 + }, + "test": { + "rmse": 0.23764734685399194, + "mae": 0.15499013662338257, + "r2": 0.6140726804733276, + "score": -0.23764734685399194 + } + }, + "time": "0:02:03.488922", + "diversity": { + "train": { + "mean_centered_corr": -0.019381657242774963, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04552893713116646 + }, + "val": { + "mean_centered_corr": -0.01903798058629036, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06320050358772278 + }, + "test": { + "mean_centered_corr": -0.018483227118849754, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08418284356594086 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 0, + 25, + 21 + ], + "head_selection_time": "0:00:00.074365" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a5b17ed41 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23937254509775088, + "mae": 0.12237884849309921, + "r2": 0.6339312791824341, + "score": -0.23937254509775088 + }, + "val": { + "rmse": 0.236445130001989, + "mae": 0.13012416660785675, + "r2": 0.6265676021575928, + "score": -0.236445130001989 + }, + "test": { + "rmse": 0.2384320235467145, + "mae": 0.14947566390037537, + "r2": 0.6115199327468872, + "score": -0.2384320235467145 + } + }, + "time": "0:02:02.511800", + "diversity": { + "train": { + "mean_centered_corr": -0.01890525035560131, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0457611083984375 + }, + "val": { + "mean_centered_corr": -0.019421136006712914, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06420256942510605 + }, + "test": { + "mean_centered_corr": -0.01877988502383232, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08486068248748779 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015361" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0.toml new file mode 100644 index 000000000..8b4c4cda1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..a464d6f76 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23134854060109078, + "mae": 0.11915364116430283, + "r2": 0.6580619812011719, + "score": -0.23134854060109078 + }, + "val": { + "rmse": 0.23526017405140137, + "mae": 0.12479399889707565, + "r2": 0.6303011775016785, + "score": -0.23526017405140137 + }, + "test": { + "rmse": 0.23237540477719482, + "mae": 0.14426252245903015, + "r2": 0.6310054659843445, + "score": -0.23237540477719482 + } + }, + "time": "0:02:02.511800", + "diversity": { + "train": { + "mean_centered_corr": -0.01890525035560131, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0457611083984375 + }, + "val": { + "mean_centered_corr": -0.019421136006712914, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06420256942510605 + }, + "test": { + "mean_centered_corr": -0.01877988502383232, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08486068248748779 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..a01a45660 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23389260398134046, + "mae": 0.12011703103780746, + "r2": 0.6505002379417419, + "score": -0.23389260398134046 + }, + "val": { + "rmse": 0.23288295595018185, + "mae": 0.1259579211473465, + "r2": 0.6377347707748413, + "score": -0.23288295595018185 + }, + "test": { + "rmse": 0.2378111877355073, + "mae": 0.15456625819206238, + "r2": 0.6135403513908386, + "score": -0.2378111877355073 + } + }, + "time": "0:02:02.511800", + "diversity": { + "train": { + "mean_centered_corr": -0.01890525035560131, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0457611083984375 + }, + "val": { + "mean_centered_corr": -0.019421136006712914, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06420256942510605 + }, + "test": { + "mean_centered_corr": -0.01877988502383232, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08486068248748779 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 16, + 19, + 10, + 25, + 1 + ], + "head_selection_time": "0:00:00.102182" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..050fbfdcd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24201867156469675, + "mae": 0.1283414661884308, + "r2": 0.6257932186126709, + "score": -0.24201867156469675 + }, + "val": { + "rmse": 0.23792789399055542, + "mae": 0.13413427770137787, + "r2": 0.6218692660331726, + "score": -0.23792789399055542 + }, + "test": { + "rmse": 0.23833634568163614, + "mae": 0.15245570242404938, + "r2": 0.6118316650390625, + "score": -0.23833634568163614 + } + }, + "time": "0:02:02.915603", + "diversity": { + "train": { + "mean_centered_corr": -0.018602978438138962, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04584116488695145 + }, + "val": { + "mean_centered_corr": -0.01837269775569439, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06368750333786011 + }, + "test": { + "mean_centered_corr": -0.019264845177531242, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08078610152006149 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015414" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0.toml new file mode 100644 index 000000000..a3f9ae4bb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..8ce5fa5b2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23062440497365552, + "mae": 0.12134777754545212, + "r2": 0.6601991653442383, + "score": -0.23062440497365552 + }, + "val": { + "rmse": 0.23522375918391628, + "mae": 0.1281193345785141, + "r2": 0.6304156184196472, + "score": -0.23522375918391628 + }, + "test": { + "rmse": 0.23361716934488255, + "mae": 0.14736692607402802, + "r2": 0.6270512938499451, + "score": -0.23361716934488255 + } + }, + "time": "0:02:02.915603", + "diversity": { + "train": { + "mean_centered_corr": -0.018602978438138962, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04584116488695145 + }, + "val": { + "mean_centered_corr": -0.01837269775569439, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06368750333786011 + }, + "test": { + "mean_centered_corr": -0.019264845177531242, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08078610152006149 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..b18c6b5b4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23198909697727824, + "mae": 0.12181763350963593, + "r2": 0.6561658382415771, + "score": -0.23198909697727824 + }, + "val": { + "rmse": 0.23351789154964117, + "mae": 0.12995123863220215, + "r2": 0.6357567310333252, + "score": -0.23351789154964117 + }, + "test": { + "rmse": 0.2420072500053204, + "mae": 0.1604735553264618, + "r2": 0.5997822284698486, + "score": -0.2420072500053204 + } + }, + "time": "0:02:02.915603", + "diversity": { + "train": { + "mean_centered_corr": -0.018602978438138962, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04584116488695145 + }, + "val": { + "mean_centered_corr": -0.01837269775569439, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06368750333786011 + }, + "test": { + "mean_centered_corr": -0.019264845177531242, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08078610152006149 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 25, + 12, + 19, + 25, + 10 + ], + "head_selection_time": "0:00:00.103588" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f3b5cf91b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24186190129822632, + "mae": 0.12565605342388153, + "r2": 0.6262778639793396, + "score": -0.24186190129822632 + }, + "val": { + "rmse": 0.23758917502158358, + "mae": 0.13536617159843445, + "r2": 0.6229451298713684, + "score": -0.23758917502158358 + }, + "test": { + "rmse": 0.23986609072189577, + "mae": 0.15381069481372833, + "r2": 0.60683274269104, + "score": -0.23986609072189577 + } + }, + "time": "0:02:02.109256", + "diversity": { + "train": { + "mean_centered_corr": -0.018595725297927856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045762546360492706 + }, + "val": { + "mean_centered_corr": -0.01878608576953411, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06204522028565407 + }, + "test": { + "mean_centered_corr": -0.019662391394376755, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08173146098852158 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015281" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0.toml new file mode 100644 index 000000000..7a5f15280 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..6896eac74 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23179295341432377, + "mae": 0.12123288959264755, + "r2": 0.6567469835281372, + "score": -0.23179295341432377 + }, + "val": { + "rmse": 0.23474049679651998, + "mae": 0.12795619666576385, + "r2": 0.6319326162338257, + "score": -0.23474049679651998 + }, + "test": { + "rmse": 0.23271906309028628, + "mae": 0.1455690711736679, + "r2": 0.6299132704734802, + "score": -0.23271906309028628 + } + }, + "time": "0:02:02.109256", + "diversity": { + "train": { + "mean_centered_corr": -0.018595725297927856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045762546360492706 + }, + "val": { + "mean_centered_corr": -0.01878608576953411, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06204522028565407 + }, + "test": { + "mean_centered_corr": -0.019662391394376755, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08173146098852158 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..607749a58 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,704 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23288190818323617, + "mae": 0.12223333865404129, + "r2": 0.6535142660140991, + "score": -0.23288190818323617 + }, + "val": { + "rmse": 0.23335307969400199, + "mae": 0.1274515837430954, + "r2": 0.6362706422805786, + "score": -0.23335307969400199 + }, + "test": { + "rmse": 0.23233217223360508, + "mae": 0.1452494114637375, + "r2": 0.6311427354812622, + "score": -0.23233217223360508 + } + }, + "time": "0:02:02.109256", + "diversity": { + "train": { + "mean_centered_corr": -0.018595725297927856, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045762546360492706 + }, + "val": { + "mean_centered_corr": -0.01878608576953411, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06204522028565407 + }, + "test": { + "mean_centered_corr": -0.019662391394376755, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08173146098852158 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 21, + 0, + 3, + 25, + 12, + 19, + 1, + 25, + 5, + 24 + ], + "head_selection_time": "0:00:00.175771" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..1977dabfd --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24408485538557934, + "mae": 0.12816353142261505, + "r2": 0.619376540184021, + "score": -0.24408485538557934 + }, + "val": { + "rmse": 0.23859870588230647, + "mae": 0.13501523435115814, + "r2": 0.6197340488433838, + "score": -0.23859870588230647 + }, + "test": { + "rmse": 0.23721550415536108, + "mae": 0.14897823333740234, + "r2": 0.6154739856719971, + "score": -0.23721550415536108 + } + }, + "time": "0:02:04.069929", + "diversity": { + "train": { + "mean_centered_corr": -0.019251784309744835, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683033004403114 + }, + "val": { + "mean_centered_corr": -0.01828719675540924, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06498972326517105 + }, + "test": { + "mean_centered_corr": -0.01782698556780815, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08568309992551804 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015231" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0.toml new file mode 100644 index 000000000..7ba39f539 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..318083570 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23224626045326932, + "mae": 0.12201064825057983, + "r2": 0.6554031372070312, + "score": -0.23224626045326932 + }, + "val": { + "rmse": 0.23523273079765322, + "mae": 0.12723346054553986, + "r2": 0.6303874254226685, + "score": -0.23523273079765322 + }, + "test": { + "rmse": 0.231476592513396, + "mae": 0.1419006884098053, + "r2": 0.6338544487953186, + "score": -0.231476592513396 + } + }, + "time": "0:02:04.069929", + "diversity": { + "train": { + "mean_centered_corr": -0.019251784309744835, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683033004403114 + }, + "val": { + "mean_centered_corr": -0.01828719675540924, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06498972326517105 + }, + "test": { + "mean_centered_corr": -0.01782698556780815, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08568309992551804 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..fea7976da --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p9_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,702 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2319716894083542, + "mae": 0.12287101149559021, + "r2": 0.6562174558639526, + "score": -0.2319716894083542 + }, + "val": { + "rmse": 0.23377952428751153, + "mae": 0.12833169102668762, + "r2": 0.6349400281906128, + "score": -0.23377952428751153 + }, + "test": { + "rmse": 0.23516218393385976, + "mae": 0.151117205619812, + "r2": 0.6221020221710205, + "score": -0.23516218393385976 + } + }, + "time": "0:02:04.069929", + "diversity": { + "train": { + "mean_centered_corr": -0.019251784309744835, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04683033004403114 + }, + "val": { + "mean_centered_corr": -0.01828719675540924, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06498972326517105 + }, + "test": { + "mean_centered_corr": -0.01782698556780815, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08568309992551804 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 0, + 21, + 28, + 25, + 12, + 17, + 19, + 18 + ], + "head_selection_time": "0:00:00.146054" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..10f8d933c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22546390849445092, + "mae": 0.12138189375400543, + "r2": 0.67523592710495, + "score": -0.22546390849445092 + }, + "val": { + "rmse": 0.2384361951499822, + "mae": 0.13219110667705536, + "r2": 0.6202518939971924, + "score": -0.2384361951499822 + }, + "test": { + "rmse": 0.23890960422803306, + "mae": 0.14420710504055023, + "r2": 0.6099621057510376, + "score": -0.23890960422803306 + } + }, + "time": "0:02:16.435477", + "diversity": { + "train": { + "mean_centered_corr": -0.017872996628284454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04516569897532463 + }, + "val": { + "mean_centered_corr": -0.01949959062039852, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06368565559387207 + }, + "test": { + "mean_centered_corr": -0.016032706946134567, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08689064532518387 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 8 + ], + "head_selection_time": "0:00:00.015277" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0.toml new file mode 100644 index 000000000..39a3eb794 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..91783d0ab --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21667307678393335, + "mae": 0.1152627021074295, + "r2": 0.7000673413276672, + "score": -0.21667307678393335 + }, + "val": { + "rmse": 0.2352393029012561, + "mae": 0.12733106315135956, + "r2": 0.6303667426109314, + "score": -0.2352393029012561 + }, + "test": { + "rmse": 0.23369680670218232, + "mae": 0.14800961315631866, + "r2": 0.6267969608306885, + "score": -0.23369680670218232 + } + }, + "time": "0:02:16.435477", + "diversity": { + "train": { + "mean_centered_corr": -0.017872996628284454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04516569897532463 + }, + "val": { + "mean_centered_corr": -0.01949959062039852, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06368565559387207 + }, + "test": { + "mean_centered_corr": -0.016032706946134567, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08689064532518387 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..cf3e69226 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2179749245264706, + "mae": 0.11718267947435379, + "r2": 0.696452260017395, + "score": -0.2179749245264706 + }, + "val": { + "rmse": 0.2338562469701412, + "mae": 0.1282421499490738, + "r2": 0.6347004175186157, + "score": -0.2338562469701412 + }, + "test": { + "rmse": 0.23357564989983745, + "mae": 0.1465408205986023, + "r2": 0.6271837949752808, + "score": -0.23357564989983745 + } + }, + "time": "0:02:16.435477", + "diversity": { + "train": { + "mean_centered_corr": -0.017872996628284454, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04516569897532463 + }, + "val": { + "mean_centered_corr": -0.01949959062039852, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06368565559387207 + }, + "test": { + "mean_centered_corr": -0.016032706946134567, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08689064532518387 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 8, + 0, + 4, + 19, + 21, + 25 + ], + "head_selection_time": "0:00:00.101719" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..5190a126a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24092140272621057, + "mae": 0.12480571866035461, + "r2": 0.6291787028312683, + "score": -0.24092140272621057 + }, + "val": { + "rmse": 0.2371704208371665, + "mae": 0.12888024747371674, + "r2": 0.6242730617523193, + "score": -0.2371704208371665 + }, + "test": { + "rmse": 0.23451416334598613, + "mae": 0.14349374175071716, + "r2": 0.624181866645813, + "score": -0.23451416334598613 + } + }, + "time": "0:02:05.620079", + "diversity": { + "train": { + "mean_centered_corr": -0.018756726756691933, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044779445976018906 + }, + "val": { + "mean_centered_corr": -0.019936703145503998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0639626681804657 + }, + "test": { + "mean_centered_corr": -0.01880541257560253, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07910130172967911 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015272" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0.toml new file mode 100644 index 000000000..7db59b766 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..c18abb04b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23337935525306813, + "mae": 0.12134772539138794, + "r2": 0.6520324349403381, + "score": -0.23337935525306813 + }, + "val": { + "rmse": 0.23530708779168627, + "mae": 0.12533041834831238, + "r2": 0.6301536560058594, + "score": -0.23530708779168627 + }, + "test": { + "rmse": 0.23121293389473055, + "mae": 0.1421053111553192, + "r2": 0.6346880793571472, + "score": -0.23121293389473055 + } + }, + "time": "0:02:05.620079", + "diversity": { + "train": { + "mean_centered_corr": -0.018756726756691933, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044779445976018906 + }, + "val": { + "mean_centered_corr": -0.019936703145503998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0639626681804657 + }, + "test": { + "mean_centered_corr": -0.01880541257560253, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07910130172967911 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..e4b9580d8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23514631827029459, + "mae": 0.12384053319692612, + "r2": 0.6467434167861938, + "score": -0.23514631827029459 + }, + "val": { + "rmse": 0.23311429304688983, + "mae": 0.12682734429836273, + "r2": 0.6370146870613098, + "score": -0.23311429304688983 + }, + "test": { + "rmse": 0.23653039079394922, + "mae": 0.15279927849769592, + "r2": 0.6176918745040894, + "score": -0.23653039079394922 + } + }, + "time": "0:02:05.620079", + "diversity": { + "train": { + "mean_centered_corr": -0.018756726756691933, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044779445976018906 + }, + "val": { + "mean_centered_corr": -0.019936703145503998, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0639626681804657 + }, + "test": { + "mean_centered_corr": -0.01880541257560253, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07910130172967911 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 0, + 19, + 1, + 25 + ], + "head_selection_time": "0:00:00.087358" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..6628130af --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24083051915130507, + "mae": 0.1239011138677597, + "r2": 0.6294584274291992, + "score": -0.24083051915130507 + }, + "val": { + "rmse": 0.23633714926648194, + "mae": 0.13169743120670319, + "r2": 0.6269086003303528, + "score": -0.23633714926648194 + }, + "test": { + "rmse": 0.23801691214143877, + "mae": 0.1493917554616928, + "r2": 0.6128714084625244, + "score": -0.23801691214143877 + } + }, + "time": "0:02:05.602883", + "diversity": { + "train": { + "mean_centered_corr": -0.019140595570206642, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044947654008865356 + }, + "val": { + "mean_centered_corr": -0.018659241497516632, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060750968754291534 + }, + "test": { + "mean_centered_corr": -0.015921765938401222, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07888734340667725 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015263" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0.toml new file mode 100644 index 000000000..c97d7875f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..c7d765f3a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23298860424389348, + "mae": 0.12124313414096832, + "r2": 0.6531966924667358, + "score": -0.23298860424389348 + }, + "val": { + "rmse": 0.234867262459557, + "mae": 0.1260262280702591, + "r2": 0.6315350532531738, + "score": -0.234867262459557 + }, + "test": { + "rmse": 0.23225197071547168, + "mae": 0.1443255990743637, + "r2": 0.6313973665237427, + "score": -0.23225197071547168 + } + }, + "time": "0:02:05.602883", + "diversity": { + "train": { + "mean_centered_corr": -0.019140595570206642, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044947654008865356 + }, + "val": { + "mean_centered_corr": -0.018659241497516632, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060750968754291534 + }, + "test": { + "mean_centered_corr": -0.015921765938401222, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07888734340667725 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..8d0d6d073 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,697 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23454354107218955, + "mae": 0.12458474934101105, + "r2": 0.6485521793365479, + "score": -0.23454354107218955 + }, + "val": { + "rmse": 0.2329376812231261, + "mae": 0.1272622048854828, + "r2": 0.637564480304718, + "score": -0.2329376812231261 + }, + "test": { + "rmse": 0.23722725849121748, + "mae": 0.15431377291679382, + "r2": 0.6154358386993408, + "score": -0.23722725849121748 + } + }, + "time": "0:02:05.602883", + "diversity": { + "train": { + "mean_centered_corr": -0.019140595570206642, + "mean_pairwise_disagreement": NaN, + "member_std": 0.044947654008865356 + }, + "val": { + "mean_centered_corr": -0.018659241497516632, + "mean_pairwise_disagreement": NaN, + "member_std": 0.060750968754291534 + }, + "test": { + "mean_centered_corr": -0.015921765938401222, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07888734340667725 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 25, + 21, + 19 + ], + "head_selection_time": "0:00:00.073012" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4a8a67898 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2246712226303735, + "mae": 0.12456091493368149, + "r2": 0.6775155067443848, + "score": -0.2246712226303735 + }, + "val": { + "rmse": 0.23738497878870984, + "mae": 0.1353979855775833, + "r2": 0.6235929727554321, + "score": -0.23738497878870984 + }, + "test": { + "rmse": 0.2429803744292108, + "mae": 0.157321497797966, + "r2": 0.5965571403503418, + "score": -0.2429803744292108 + } + }, + "time": "0:02:14.535269", + "diversity": { + "train": { + "mean_centered_corr": -0.01842476613819599, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04623854160308838 + }, + "val": { + "mean_centered_corr": -0.018401212990283966, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06464710086584091 + }, + "test": { + "mean_centered_corr": -0.016835635527968407, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09058339893817902 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015185" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0.toml new file mode 100644 index 000000000..06fea7ad1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..85bbd92dc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21620215214607402, + "mae": 0.11695478111505508, + "r2": 0.7013696432113647, + "score": -0.21620215214607402 + }, + "val": { + "rmse": 0.23505533340532211, + "mae": 0.12693415582180023, + "r2": 0.6309446692466736, + "score": -0.23505533340532211 + }, + "test": { + "rmse": 0.2334740169208309, + "mae": 0.1462247669696808, + "r2": 0.6275081634521484, + "score": -0.2334740169208309 + } + }, + "time": "0:02:14.535269", + "diversity": { + "train": { + "mean_centered_corr": -0.01842476613819599, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04623854160308838 + }, + "val": { + "mean_centered_corr": -0.018401212990283966, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06464710086584091 + }, + "test": { + "mean_centered_corr": -0.016835635527968407, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09058339893817902 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..6fdb8ee65 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,699 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22155602825004872, + "mae": 0.1207282543182373, + "r2": 0.6863964200019836, + "score": -0.22155602825004872 + }, + "val": { + "rmse": 0.23371274685689286, + "mae": 0.1293759047985077, + "r2": 0.6351485848426819, + "score": -0.23371274685689286 + }, + "test": { + "rmse": 0.23853136443639264, + "mae": 0.157401904463768, + "r2": 0.6111961603164673, + "score": -0.23853136443639264 + } + }, + "time": "0:02:14.535269", + "diversity": { + "train": { + "mean_centered_corr": -0.01842476613819599, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04623854160308838 + }, + "val": { + "mean_centered_corr": -0.018401212990283966, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06464710086584091 + }, + "test": { + "mean_centered_corr": -0.016835635527968407, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09058339893817902 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 25, + 28, + 22, + 6, + 1 + ], + "head_selection_time": "0:00:00.101402" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..f313e0843 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23852410213626124, + "mae": 0.12188179790973663, + "r2": 0.6365216970443726, + "score": -0.23852410213626124 + }, + "val": { + "rmse": 0.23674080812018197, + "mae": 0.12981633841991425, + "r2": 0.6256330013275146, + "score": -0.23674080812018197 + }, + "test": { + "rmse": 0.24022462096222602, + "mae": 0.15470069646835327, + "r2": 0.6056565642356873, + "score": -0.24022462096222602 + } + }, + "time": "0:02:06.448494", + "diversity": { + "train": { + "mean_centered_corr": -0.018909571692347527, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04599001631140709 + }, + "val": { + "mean_centered_corr": -0.018043946474790573, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06483229994773865 + }, + "test": { + "mean_centered_corr": -0.018734198063611984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08063172549009323 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015132" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0.toml new file mode 100644 index 000000000..a460d2818 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..5ef08a624 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2299791404863945, + "mae": 0.12113875150680542, + "r2": 0.6620979905128479, + "score": -0.2299791404863945 + }, + "val": { + "rmse": 0.23498608081258596, + "mae": 0.12758168578147888, + "r2": 0.6311621069908142, + "score": -0.23498608081258596 + }, + "test": { + "rmse": 0.23378104608060551, + "mae": 0.14829501509666443, + "r2": 0.6265279054641724, + "score": -0.23378104608060551 + } + }, + "time": "0:02:06.448494", + "diversity": { + "train": { + "mean_centered_corr": -0.018909571692347527, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04599001631140709 + }, + "val": { + "mean_centered_corr": -0.018043946474790573, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06483229994773865 + }, + "test": { + "mean_centered_corr": -0.018734198063611984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08063172549009323 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..2e44f4c73 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,700 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23242986489034317, + "mae": 0.12031129002571106, + "r2": 0.6548580527305603, + "score": -0.23242986489034317 + }, + "val": { + "rmse": 0.2333506132153353, + "mae": 0.1259656697511673, + "r2": 0.6362783908843994, + "score": -0.2333506132153353 + }, + "test": { + "rmse": 0.23993873237607374, + "mae": 0.15797272324562073, + "r2": 0.6065946221351624, + "score": -0.23993873237607374 + } + }, + "time": "0:02:06.448494", + "diversity": { + "train": { + "mean_centered_corr": -0.018909571692347527, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04599001631140709 + }, + "val": { + "mean_centered_corr": -0.018043946474790573, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06483229994773865 + }, + "test": { + "mean_centered_corr": -0.018734198063611984, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08063172549009323 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 0, + 21, + 25, + 10, + 20, + 1 + ], + "head_selection_time": "0:00:00.116624" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..570f97feb --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22891393817840822, + "mae": 0.1278449147939682, + "r2": 0.6652208566665649, + "score": -0.22891393817840822 + }, + "val": { + "rmse": 0.24086258324668144, + "mae": 0.14345626533031464, + "r2": 0.6124837398529053, + "score": -0.24086258324668144 + }, + "test": { + "rmse": 0.2704202668685561, + "mae": 0.193245068192482, + "r2": 0.5002899765968323, + "score": -0.2704202668685561 + } + }, + "time": "0:02:18.381520", + "diversity": { + "train": { + "mean_centered_corr": -0.018370768055319786, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04778972268104553 + }, + "val": { + "mean_centered_corr": -0.01997104473412037, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06924484670162201 + }, + "test": { + "mean_centered_corr": -0.018939772620797157, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08999743312597275 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015255" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0.toml new file mode 100644 index 000000000..b006acb29 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..1c243c72c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2169704086402944, + "mae": 0.11638687551021576, + "r2": 0.6992435455322266, + "score": -0.2169704086402944 + }, + "val": { + "rmse": 0.23478007280066537, + "mae": 0.12811791896820068, + "r2": 0.6318085193634033, + "score": -0.23478007280066537 + }, + "test": { + "rmse": 0.2351072792694515, + "mae": 0.14944614470005035, + "r2": 0.6222784519195557, + "score": -0.2351072792694515 + } + }, + "time": "0:02:18.381520", + "diversity": { + "train": { + "mean_centered_corr": -0.018370768055319786, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04778972268104553 + }, + "val": { + "mean_centered_corr": -0.01997104473412037, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06924484670162201 + }, + "test": { + "mean_centered_corr": -0.018939772620797157, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08999743312597275 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..853f683ed --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,707 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21863364633384474, + "mae": 0.11706539988517761, + "r2": 0.6946148872375488, + "score": -0.21863364633384474 + }, + "val": { + "rmse": 0.23354353444721934, + "mae": 0.12788079679012299, + "r2": 0.6356767416000366, + "score": -0.23354353444721934 + }, + "test": { + "rmse": 0.2390892602865205, + "mae": 0.15497441589832306, + "r2": 0.6093752980232239, + "score": -0.2390892602865205 + } + }, + "time": "0:02:18.381520", + "diversity": { + "train": { + "mean_centered_corr": -0.018370768055319786, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04778972268104553 + }, + "val": { + "mean_centered_corr": -0.01997104473412037, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06924484670162201 + }, + "test": { + "mean_centered_corr": -0.018939772620797157, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08999743312597275 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25, + 7, + 10, + 22, + 21, + 14, + 31, + 12, + 6, + 10, + 22, + 27, + 19, + 10 + ], + "head_selection_time": "0:00:00.219853" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..35188f4ee --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22951726573518907, + "mae": 0.12178733944892883, + "r2": 0.6634538769721985, + "score": -0.22951726573518907 + }, + "val": { + "rmse": 0.237046057654127, + "mae": 0.13444463908672333, + "r2": 0.6246670484542847, + "score": -0.237046057654127 + }, + "test": { + "rmse": 0.2530936074511429, + "mae": 0.17497020959854126, + "r2": 0.5622743964195251, + "score": -0.2530936074511429 + } + }, + "time": "0:02:19.714907", + "diversity": { + "train": { + "mean_centered_corr": -0.018565235659480095, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782221093773842 + }, + "val": { + "mean_centered_corr": -0.01974944956600666, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06836902350187302 + }, + "test": { + "mean_centered_corr": -0.01930362544953823, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09177923202514648 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015486" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0.toml new file mode 100644 index 000000000..9206c4214 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..1d06e85e1 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21750608206883032, + "mae": 0.11355320364236832, + "r2": 0.6977567076683044, + "score": -0.21750608206883032 + }, + "val": { + "rmse": 0.2350854992177386, + "mae": 0.12681186199188232, + "r2": 0.6308499574661255, + "score": -0.2350854992177386 + }, + "test": { + "rmse": 0.23319390245153368, + "mae": 0.14503715932369232, + "r2": 0.628401517868042, + "score": -0.23319390245153368 + } + }, + "time": "0:02:19.714907", + "diversity": { + "train": { + "mean_centered_corr": -0.018565235659480095, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782221093773842 + }, + "val": { + "mean_centered_corr": -0.01974944956600666, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06836902350187302 + }, + "test": { + "mean_centered_corr": -0.01930362544953823, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09177923202514648 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..95d9be15f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,701 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22152409575796225, + "mae": 0.1159883365035057, + "r2": 0.6864867806434631, + "score": -0.22152409575796225 + }, + "val": { + "rmse": 0.23381808394805512, + "mae": 0.12612800300121307, + "r2": 0.6348196268081665, + "score": -0.23381808394805512 + }, + "test": { + "rmse": 0.230425386066968, + "mae": 0.13822335004806519, + "r2": 0.6371724605560303, + "score": -0.230425386066968 + } + }, + "time": "0:02:19.714907", + "diversity": { + "train": { + "mean_centered_corr": -0.018565235659480095, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04782221093773842 + }, + "val": { + "mean_centered_corr": -0.01974944956600666, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06836902350187302 + }, + "test": { + "mean_centered_corr": -0.01930362544953823, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09177923202514648 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25, + 9, + 5, + 0, + 19, + 22, + 29, + 3 + ], + "head_selection_time": "0:00:00.131585" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..3aada2516 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2429541865580833, + "mae": 0.1267397403717041, + "r2": 0.6228946447372437, + "score": -0.2429541865580833 + }, + "val": { + "rmse": 0.23844788931164226, + "mae": 0.13394595682621002, + "r2": 0.6202146410942078, + "score": -0.23844788931164226 + }, + "test": { + "rmse": 0.239992780080121, + "mae": 0.152608722448349, + "r2": 0.6064173579216003, + "score": -0.239992780080121 + } + }, + "time": "0:02:02.867107", + "diversity": { + "train": { + "mean_centered_corr": -0.019329514354467392, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04684953764081001 + }, + "val": { + "mean_centered_corr": -0.01879928633570671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0651373490691185 + }, + "test": { + "mean_centered_corr": -0.02050378918647766, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08481118083000183 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015382" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0.toml new file mode 100644 index 000000000..8d4da8b93 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..3b3aa2a82 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23163036400903525, + "mae": 0.12018397450447083, + "r2": 0.6572283506393433, + "score": -0.23163036400903525 + }, + "val": { + "rmse": 0.23460196790457225, + "mae": 0.1260966658592224, + "r2": 0.632366955280304, + "score": -0.23460196790457225 + }, + "test": { + "rmse": 0.23160625447220415, + "mae": 0.1424369513988495, + "r2": 0.6334441900253296, + "score": -0.23160625447220415 + } + }, + "time": "0:02:02.867107", + "diversity": { + "train": { + "mean_centered_corr": -0.019329514354467392, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04684953764081001 + }, + "val": { + "mean_centered_corr": -0.01879928633570671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0651373490691185 + }, + "test": { + "mean_centered_corr": -0.02050378918647766, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08481118083000183 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..bbf779053 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,700 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23151798153687342, + "mae": 0.12077950686216354, + "r2": 0.6575609445571899, + "score": -0.23151798153687342 + }, + "val": { + "rmse": 0.23278091651666263, + "mae": 0.12862801551818848, + "r2": 0.6380521655082703, + "score": -0.23278091651666263 + }, + "test": { + "rmse": 0.23692641701684622, + "mae": 0.15434525907039642, + "r2": 0.6164106130599976, + "score": -0.23692641701684622 + } + }, + "time": "0:02:02.867107", + "diversity": { + "train": { + "mean_centered_corr": -0.019329514354467392, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04684953764081001 + }, + "val": { + "mean_centered_corr": -0.01879928633570671, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0651373490691185 + }, + "test": { + "mean_centered_corr": -0.02050378918647766, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08481118083000183 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 25, + 12, + 18, + 29, + 25, + 19 + ], + "head_selection_time": "0:00:00.116591" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..a9c72a6c4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24471092752622822, + "mae": 0.12746074795722961, + "r2": 0.6174214482307434, + "score": -0.24471092752622822 + }, + "val": { + "rmse": 0.23860736324692997, + "mae": 0.13630448281764984, + "r2": 0.6197065114974976, + "score": -0.23860736324692997 + }, + "test": { + "rmse": 0.24541511515454548, + "mae": 0.16491781175136566, + "r2": 0.5884314179420471, + "score": -0.24541511515454548 + } + }, + "time": "0:02:05.625831", + "diversity": { + "train": { + "mean_centered_corr": -0.019371401518583298, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046141691505908966 + }, + "val": { + "mean_centered_corr": -0.01898176036775112, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06393436342477798 + }, + "test": { + "mean_centered_corr": -0.019255701452493668, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08550143241882324 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015058" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0.toml new file mode 100644 index 000000000..55eb6a51e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..d16c1f969 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23234667483081278, + "mae": 0.12070997804403305, + "r2": 0.6551050543785095, + "score": -0.23234667483081278 + }, + "val": { + "rmse": 0.23525231195767085, + "mae": 0.1279769092798233, + "r2": 0.6303259134292603, + "score": -0.23525231195767085 + }, + "test": { + "rmse": 0.23376059267196012, + "mae": 0.14717192947864532, + "r2": 0.6265932321548462, + "score": -0.23376059267196012 + } + }, + "time": "0:02:05.625831", + "diversity": { + "train": { + "mean_centered_corr": -0.019371401518583298, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046141691505908966 + }, + "val": { + "mean_centered_corr": -0.01898176036775112, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06393436342477798 + }, + "test": { + "mean_centered_corr": -0.019255701452493668, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08550143241882324 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..96be49ce3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,704 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23244703781733334, + "mae": 0.12160777300596237, + "r2": 0.6548070907592773, + "score": -0.23244703781733334 + }, + "val": { + "rmse": 0.23326554764868923, + "mae": 0.1277935653924942, + "r2": 0.6365435123443604, + "score": -0.23326554764868923 + }, + "test": { + "rmse": 0.24138003765619687, + "mae": 0.16120430827140808, + "r2": 0.6018540859222412, + "score": -0.24138003765619687 + } + }, + "time": "0:02:05.625831", + "diversity": { + "train": { + "mean_centered_corr": -0.019371401518583298, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046141691505908966 + }, + "val": { + "mean_centered_corr": -0.01898176036775112, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06393436342477798 + }, + "test": { + "mean_centered_corr": -0.019255701452493668, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08550143241882324 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 25, + 12, + 18, + 20, + 21, + 25, + 12, + 25, + 19, + 16 + ], + "head_selection_time": "0:00:00.173565" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..9065a10a8 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23908905773123074, + "mae": 0.13352937996387482, + "r2": 0.6347978711128235, + "score": -0.23908905773123074 + }, + "val": { + "rmse": 0.23745188445279078, + "mae": 0.13348934054374695, + "r2": 0.6233807802200317, + "score": -0.23745188445279078 + }, + "test": { + "rmse": 0.23531645200331283, + "mae": 0.14286726713180542, + "r2": 0.6216060519218445, + "score": -0.23531645200331283 + } + }, + "time": "0:02:05.428840", + "diversity": { + "train": { + "mean_centered_corr": -0.01883065141737461, + "mean_pairwise_disagreement": NaN, + "member_std": 0.043418336659669876 + }, + "val": { + "mean_centered_corr": -0.019489828497171402, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058777473866939545 + }, + "test": { + "mean_centered_corr": -0.017089199274778366, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07344797253608704 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015304" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0.toml new file mode 100644 index 000000000..a47046c92 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..484eee180 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.2330807634805131, + "mae": 0.12179981172084808, + "r2": 0.6529222726821899, + "score": -0.2330807634805131 + }, + "val": { + "rmse": 0.23558367762733304, + "mae": 0.12585210800170898, + "r2": 0.6292837262153625, + "score": -0.23558367762733304 + }, + "test": { + "rmse": 0.2306278293973631, + "mae": 0.14087699353694916, + "r2": 0.6365346312522888, + "score": -0.2306278293973631 + } + }, + "time": "0:02:05.428840", + "diversity": { + "train": { + "mean_centered_corr": -0.01883065141737461, + "mean_pairwise_disagreement": NaN, + "member_std": 0.043418336659669876 + }, + "val": { + "mean_centered_corr": -0.019489828497171402, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058777473866939545 + }, + "test": { + "mean_centered_corr": -0.017089199274778366, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07344797253608704 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..1d4057f55 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,703 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23343546426030956, + "mae": 0.12315606325864792, + "r2": 0.6518651247024536, + "score": -0.23343546426030956 + }, + "val": { + "rmse": 0.2330749056995843, + "mae": 0.12734414637088776, + "r2": 0.6371372938156128, + "score": -0.2330749056995843 + }, + "test": { + "rmse": 0.23692981324824752, + "mae": 0.1541774570941925, + "r2": 0.6163996458053589, + "score": -0.23692981324824752 + } + }, + "time": "0:02:05.428840", + "diversity": { + "train": { + "mean_centered_corr": -0.01883065141737461, + "mean_pairwise_disagreement": NaN, + "member_std": 0.043418336659669876 + }, + "val": { + "mean_centered_corr": -0.019489828497171402, + "mean_pairwise_disagreement": NaN, + "member_std": 0.058777473866939545 + }, + "test": { + "mean_centered_corr": -0.017089199274778366, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07344797253608704 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 0, + 25, + 21, + 25, + 4, + 19, + 6, + 1, + 25, + 22 + ], + "head_selection_time": "0:00:00.158945" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..cc7e36898 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23844750654589067, + "mae": 0.12963111698627472, + "r2": 0.6367551684379578, + "score": -0.23844750654589067 + }, + "val": { + "rmse": 0.23707190033649148, + "mae": 0.1328258216381073, + "r2": 0.6245851516723633, + "score": -0.23707190033649148 + }, + "test": { + "rmse": 0.2397359546762736, + "mae": 0.1563534289598465, + "r2": 0.6072592735290527, + "score": -0.2397359546762736 + } + }, + "time": "0:02:03.794551", + "diversity": { + "train": { + "mean_centered_corr": -0.018837179988622665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04427312687039375 + }, + "val": { + "mean_centered_corr": -0.020433422178030014, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05948192626237869 + }, + "test": { + "mean_centered_corr": -0.018294233828783035, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07480351626873016 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 0 + ], + "head_selection_time": "0:00:00.015171" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0.toml new file mode 100644 index 000000000..979ff2558 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..27d9cb616 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23271073895226718, + "mae": 0.12118933349847794, + "r2": 0.6540234088897705, + "score": -0.23271073895226718 + }, + "val": { + "rmse": 0.23476767229656906, + "mae": 0.12631559371948242, + "r2": 0.6318474411964417, + "score": -0.23476767229656906 + }, + "test": { + "rmse": 0.23170820829575262, + "mae": 0.14362695813179016, + "r2": 0.6331213712692261, + "score": -0.23170820829575262 + } + }, + "time": "0:02:03.794551", + "diversity": { + "train": { + "mean_centered_corr": -0.018837179988622665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04427312687039375 + }, + "val": { + "mean_centered_corr": -0.020433422178030014, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05948192626237869 + }, + "test": { + "mean_centered_corr": -0.018294233828783035, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07480351626873016 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..24d1febd6 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,700 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23438370211658663, + "mae": 0.12246029078960419, + "r2": 0.6490310430526733, + "score": -0.23438370211658663 + }, + "val": { + "rmse": 0.23225748836355237, + "mae": 0.12540538609027863, + "r2": 0.6396780610084534, + "score": -0.23225748836355237 + }, + "test": { + "rmse": 0.2373654401585729, + "mae": 0.1536472588777542, + "r2": 0.6149877309799194, + "score": -0.2373654401585729 + } + }, + "time": "0:02:03.794551", + "diversity": { + "train": { + "mean_centered_corr": -0.018837179988622665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04427312687039375 + }, + "val": { + "mean_centered_corr": -0.020433422178030014, + "mean_pairwise_disagreement": NaN, + "member_std": 0.05948192626237869 + }, + "test": { + "mean_centered_corr": -0.018294233828783035, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07480351626873016 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 0, + 4, + 25, + 21, + 19, + 25, + 21 + ], + "head_selection_time": "0:00:00.130562" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..2415d6d35 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2323950263151425, + "mae": 0.12666819989681244, + "r2": 0.6549615263938904, + "score": -0.2323950263151425 + }, + "val": { + "rmse": 0.2386455487030765, + "mae": 0.14065667986869812, + "r2": 0.6195847392082214, + "score": -0.2386455487030765 + }, + "test": { + "rmse": 0.27701433524933666, + "mae": 0.19954755902290344, + "r2": 0.4756224751472473, + "score": -0.27701433524933666 + } + }, + "time": "0:02:18.086910", + "diversity": { + "train": { + "mean_centered_corr": -0.01866568624973297, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04555417224764824 + }, + "val": { + "mean_centered_corr": -0.01869249902665615, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06710393726825714 + }, + "test": { + "mean_centered_corr": -0.01764187403023243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08898688107728958 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015682" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0.toml new file mode 100644 index 000000000..5c0b05d96 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.95 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..5fce97243 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21881660230888528, + "mae": 0.11528468877077103, + "r2": 0.6941035985946655, + "score": -0.21881660230888528 + }, + "val": { + "rmse": 0.2351196777064841, + "mae": 0.12626346945762634, + "r2": 0.630742609500885, + "score": -0.2351196777064841 + }, + "test": { + "rmse": 0.23285365668962787, + "mae": 0.14447025954723358, + "r2": 0.6294850707054138, + "score": -0.23285365668962787 + } + }, + "time": "0:02:18.086910", + "diversity": { + "train": { + "mean_centered_corr": -0.01866568624973297, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04555417224764824 + }, + "val": { + "mean_centered_corr": -0.01869249902665615, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06710393726825714 + }, + "test": { + "mean_centered_corr": -0.01764187403023243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08898688107728958 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..48a5a1aa2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,698 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.95, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9515306353569031, + "min_features_per_member": 362.0, + "mean_features_per_member": 373.0, + "max_features_per_member": 384.0, + "min_members_per_feature": 26.0, + "mean_members_per_feature": 30.4489803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9072567877562938, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22086028183547482, + "mae": 0.11741601675748825, + "r2": 0.6883629560470581, + "score": -0.22086028183547482 + }, + "val": { + "rmse": 0.23375630575487258, + "mae": 0.12780286371707916, + "r2": 0.6350125074386597, + "score": -0.23375630575487258 + }, + "test": { + "rmse": 0.23772689559358076, + "mae": 0.15195266902446747, + "r2": 0.6138142347335815, + "score": -0.23772689559358076 + } + }, + "time": "0:02:18.086910", + "diversity": { + "train": { + "mean_centered_corr": -0.01866568624973297, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04555417224764824 + }, + "val": { + "mean_centered_corr": -0.01869249902665615, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06710393726825714 + }, + "test": { + "mean_centered_corr": -0.01764187403023243, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08898688107728958 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25, + 8, + 21, + 7, + 12 + ], + "head_selection_time": "0:00:00.090438" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..4bc6f4ede --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.23131872489766006, + "mae": 0.1263350546360016, + "r2": 0.6581500768661499, + "score": -0.23131872489766006 + }, + "val": { + "rmse": 0.23771184363825437, + "mae": 0.13543455302715302, + "r2": 0.622555673122406, + "score": -0.23771184363825437 + }, + "test": { + "rmse": 0.2596801100779992, + "mae": 0.17902591824531555, + "r2": 0.53919517993927, + "score": -0.2596801100779992 + } + }, + "time": "0:02:19.164877", + "diversity": { + "train": { + "mean_centered_corr": -0.01825721003115177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045883432030677795 + }, + "val": { + "mean_centered_corr": -0.019998809322714806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0684179738163948 + }, + "test": { + "mean_centered_corr": -0.019780565053224564, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08797822892665863 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015296" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0.toml new file mode 100644 index 000000000..996f958aa --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..822ae80ed --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21707002083369387, + "mae": 0.11528186500072479, + "r2": 0.6989673376083374, + "score": -0.21707002083369387 + }, + "val": { + "rmse": 0.23556640917246102, + "mae": 0.12705539166927338, + "r2": 0.6293380260467529, + "score": -0.23556640917246102 + }, + "test": { + "rmse": 0.23394963288342588, + "mae": 0.14626435935497284, + "r2": 0.6259890794754028, + "score": -0.23394963288342588 + } + }, + "time": "0:02:19.164877", + "diversity": { + "train": { + "mean_centered_corr": -0.01825721003115177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045883432030677795 + }, + "val": { + "mean_centered_corr": -0.019998809322714806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0684179738163948 + }, + "test": { + "mean_centered_corr": -0.019780565053224564, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08797822892665863 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..c55531ba4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,702 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.22058343595772106, + "mae": 0.11804859340190887, + "r2": 0.689143717288971, + "score": -0.22058343595772106 + }, + "val": { + "rmse": 0.2337060760344375, + "mae": 0.12719006836414337, + "r2": 0.6351693868637085, + "score": -0.2337060760344375 + }, + "test": { + "rmse": 0.23617809041406845, + "mae": 0.14892512559890747, + "r2": 0.6188299059867859, + "score": -0.23617809041406845 + } + }, + "time": "0:02:19.164877", + "diversity": { + "train": { + "mean_centered_corr": -0.01825721003115177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.045883432030677795 + }, + "val": { + "mean_centered_corr": -0.019998809322714806, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0684179738163948 + }, + "test": { + "mean_centered_corr": -0.019780565053224564, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08797822892665863 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25, + 4, + 19, + 0, + 16, + 22, + 10, + 20, + 4 + ], + "head_selection_time": "0:00:00.146000" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..bda9dcf87 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23600635218658622, + "mae": 0.12171389162540436, + "r2": 0.6441546678543091, + "score": -0.23600635218658622 + }, + "val": { + "rmse": 0.23788508328903052, + "mae": 0.13621799647808075, + "r2": 0.6220053434371948, + "score": -0.23788508328903052 + }, + "test": { + "rmse": 0.2505198973693071, + "mae": 0.171568363904953, + "r2": 0.5711315870285034, + "score": -0.2505198973693071 + } + }, + "time": "0:02:01.587396", + "diversity": { + "train": { + "mean_centered_corr": -0.01862383633852005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04512369632720947 + }, + "val": { + "mean_centered_corr": -0.01866878941655159, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06477370113134384 + }, + "test": { + "mean_centered_corr": -0.019034983590245247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08236686140298843 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015370" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0.toml new file mode 100644 index 000000000..c1d2bcf74 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..b140bc931 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23003808694853833, + "mae": 0.12140953540802002, + "r2": 0.6619247198104858, + "score": -0.23003808694853833 + }, + "val": { + "rmse": 0.23502662197192856, + "mae": 0.12718608975410461, + "r2": 0.6310348510742188, + "score": -0.23502662197192856 + }, + "test": { + "rmse": 0.23487478850600307, + "mae": 0.149356409907341, + "r2": 0.623025119304657, + "score": -0.23487478850600307 + } + }, + "time": "0:02:01.587396", + "diversity": { + "train": { + "mean_centered_corr": -0.01862383633852005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04512369632720947 + }, + "val": { + "mean_centered_corr": -0.01866878941655159, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06477370113134384 + }, + "test": { + "mean_centered_corr": -0.019034983590245247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08236686140298843 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..19dc64a59 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,701 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23224673364044418, + "mae": 0.12295585125684738, + "r2": 0.6554017066955566, + "score": -0.23224673364044418 + }, + "val": { + "rmse": 0.23340701642091277, + "mae": 0.12782487273216248, + "r2": 0.6361024975776672, + "score": -0.23340701642091277 + }, + "test": { + "rmse": 0.24153220831875505, + "mae": 0.16074170172214508, + "r2": 0.6013518571853638, + "score": -0.24153220831875505 + } + }, + "time": "0:02:01.587396", + "diversity": { + "train": { + "mean_centered_corr": -0.01862383633852005, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04512369632720947 + }, + "val": { + "mean_centered_corr": -0.01866878941655159, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06477370113134384 + }, + "test": { + "mean_centered_corr": -0.019034983590245247, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08236686140298843 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 21, + 25, + 10, + 19, + 21, + 0, + 16 + ], + "head_selection_time": "0:00:00.133352" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..62af2190c --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2242404395588436, + "mae": 0.13221053779125214, + "r2": 0.6787509918212891, + "score": -0.2242404395588436 + }, + "val": { + "rmse": 0.23901159112197984, + "mae": 0.13654795289039612, + "r2": 0.6184168457984924, + "score": -0.23901159112197984 + }, + "test": { + "rmse": 0.24670214607411306, + "mae": 0.1664496809244156, + "r2": 0.5841033458709717, + "score": -0.24670214607411306 + } + }, + "time": "0:02:18.814512", + "diversity": { + "train": { + "mean_centered_corr": -0.017886240035295486, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046517129987478256 + }, + "val": { + "mean_centered_corr": -0.019535018131136894, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06728510558605194 + }, + "test": { + "mean_centered_corr": -0.018488358706235886, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08319593966007233 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 21 + ], + "head_selection_time": "0:00:00.015378" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0.toml new file mode 100644 index 000000000..85883a50e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..e87175c62 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21727375472083588, + "mae": 0.1175784170627594, + "r2": 0.6984020471572876, + "score": -0.21727375472083588 + }, + "val": { + "rmse": 0.2349593190227014, + "mae": 0.12777242064476013, + "r2": 0.6312460899353027, + "score": -0.2349593190227014 + }, + "test": { + "rmse": 0.23427020750192581, + "mae": 0.14806360006332397, + "r2": 0.6249633431434631, + "score": -0.23427020750192581 + } + }, + "time": "0:02:18.814512", + "diversity": { + "train": { + "mean_centered_corr": -0.017886240035295486, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046517129987478256 + }, + "val": { + "mean_centered_corr": -0.019535018131136894, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06728510558605194 + }, + "test": { + "mean_centered_corr": -0.018488358706235886, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08319593966007233 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..6c0f527a4 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p975_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,701 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.218914778840735, + "mae": 0.11776337772607803, + "r2": 0.6938289999961853, + "score": -0.218914778840735 + }, + "val": { + "rmse": 0.2334202712313093, + "mae": 0.1272968202829361, + "r2": 0.6360611915588379, + "score": -0.2334202712313093 + }, + "test": { + "rmse": 0.2351349272835184, + "mae": 0.1497408002614975, + "r2": 0.6221896409988403, + "score": -0.2351349272835184 + } + }, + "time": "0:02:18.814512", + "diversity": { + "train": { + "mean_centered_corr": -0.017886240035295486, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046517129987478256 + }, + "val": { + "mean_centered_corr": -0.019535018131136894, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06728510558605194 + }, + "test": { + "mean_centered_corr": -0.018488358706235886, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08319593966007233 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 21, + 7, + 10, + 25, + 19, + 10, + 22, + 6 + ], + "head_selection_time": "0:00:00.131078" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..b20a2e0dc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24215561133839658, + "mae": 0.1261674165725708, + "r2": 0.625369668006897, + "score": -0.24215561133839658 + }, + "val": { + "rmse": 0.23901917370395387, + "mae": 0.1357598900794983, + "r2": 0.6183926463127136, + "score": -0.23901917370395387 + }, + "test": { + "rmse": 0.23709576833229662, + "mae": 0.1498398780822754, + "r2": 0.6158620715141296, + "score": -0.23709576833229662 + } + }, + "time": "0:02:05.583255", + "diversity": { + "train": { + "mean_centered_corr": -0.01902354508638382, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04608777537941933 + }, + "val": { + "mean_centered_corr": -0.01785912550985813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06531322002410889 + }, + "test": { + "mean_centered_corr": -0.020982077345252037, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0832035094499588 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015284" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0.toml new file mode 100644 index 000000000..f061a28d3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..a9bf21b1f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23131163072255229, + "mae": 0.12073468416929245, + "r2": 0.6581710577011108, + "score": -0.23131163072255229 + }, + "val": { + "rmse": 0.23544191074619497, + "mae": 0.1270890235900879, + "r2": 0.6297297477722168, + "score": -0.23544191074619497 + }, + "test": { + "rmse": 0.23248160420213973, + "mae": 0.14418110251426697, + "r2": 0.6306681632995605, + "score": -0.23248160420213973 + } + }, + "time": "0:02:05.583255", + "diversity": { + "train": { + "mean_centered_corr": -0.01902354508638382, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04608777537941933 + }, + "val": { + "mean_centered_corr": -0.01785912550985813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06531322002410889 + }, + "test": { + "mean_centered_corr": -0.020982077345252037, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0832035094499588 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..84b3884bc --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,697 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23320069974727647, + "mae": 0.1219385489821434, + "r2": 0.6525650024414062, + "score": -0.23320069974727647 + }, + "val": { + "rmse": 0.23354370990991408, + "mae": 0.12699057161808014, + "r2": 0.6356761455535889, + "score": -0.23354370990991408 + }, + "test": { + "rmse": 0.23867420719836574, + "mae": 0.15507498383522034, + "r2": 0.6107302904129028, + "score": -0.23867420719836574 + } + }, + "time": "0:02:05.583255", + "diversity": { + "train": { + "mean_centered_corr": -0.01902354508638382, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04608777537941933 + }, + "val": { + "mean_centered_corr": -0.01785912550985813, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06531322002410889 + }, + "test": { + "mean_centered_corr": -0.020982077345252037, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0832035094499588 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 0, + 25, + 12 + ], + "head_selection_time": "0:00:00.073888" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-best-head-evaluation/0/report.json new file mode 100644 index 000000000..18484cab2 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24348651999656226, + "mae": 0.12649968266487122, + "r2": 0.6212403178215027, + "score": -0.24348651999656226 + }, + "val": { + "rmse": 0.238198961838987, + "mae": 0.13287700712680817, + "r2": 0.6210072040557861, + "score": -0.238198961838987 + }, + "test": { + "rmse": 0.23998827851003937, + "mae": 0.15366043150424957, + "r2": 0.6064320802688599, + "score": -0.23998827851003937 + } + }, + "time": "0:02:01.502226", + "diversity": { + "train": { + "mean_centered_corr": -0.01856936514377594, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046877678483724594 + }, + "val": { + "mean_centered_corr": -0.01878707855939865, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06156620755791664 + }, + "test": { + "mean_centered_corr": -0.02026924304664135, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08082668483257294 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015291" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0.toml new file mode 100644 index 000000000..824374b5d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.005 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0/report.json new file mode 100644 index 000000000..8ee5f612f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.230838543618499, + "mae": 0.12101432681083679, + "r2": 0.6595678329467773, + "score": -0.230838543618499 + }, + "val": { + "rmse": 0.23458071263974148, + "mae": 0.1285729557275772, + "r2": 0.6324335336685181, + "score": -0.23458071263974148 + }, + "test": { + "rmse": 0.23412343670770672, + "mae": 0.14895197749137878, + "r2": 0.625433087348938, + "score": -0.23412343670770672 + } + }, + "time": "0:02:01.502226", + "diversity": { + "train": { + "mean_centered_corr": -0.01856936514377594, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046877678483724594 + }, + "val": { + "mean_centered_corr": -0.01878707855939865, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06156620755791664 + }, + "test": { + "mean_centered_corr": -0.02026924304664135, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08082668483257294 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..7ab593e1f --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p005-greedy-heads-evaluation/0/report.json @@ -0,0 +1,704 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.005, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23149574309806026, + "mae": 0.121489517390728, + "r2": 0.6576266884803772, + "score": -0.23149574309806026 + }, + "val": { + "rmse": 0.2329018389418362, + "mae": 0.12706120312213898, + "r2": 0.6376760005950928, + "score": -0.2329018389418362 + }, + "test": { + "rmse": 0.23679443729395636, + "mae": 0.1538662314414978, + "r2": 0.6168378591537476, + "score": -0.23679443729395636 + } + }, + "time": "0:02:01.502226", + "diversity": { + "train": { + "mean_centered_corr": -0.01856936514377594, + "mean_pairwise_disagreement": NaN, + "member_std": 0.046877678483724594 + }, + "val": { + "mean_centered_corr": -0.01878707855939865, + "mean_pairwise_disagreement": NaN, + "member_std": 0.06156620755791664 + }, + "test": { + "mean_centered_corr": -0.02026924304664135, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08082668483257294 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 4, + 10, + 25, + 12, + 21, + 3, + 25, + 20, + 19, + 12, + 24 + ], + "head_selection_time": "0:00:00.175097" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-best-head-evaluation/0/report.json new file mode 100644 index 000000000..8fd234997 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-best-head-evaluation/0/report.json @@ -0,0 +1,694 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2295723551876263, + "mae": 0.1215803325176239, + "r2": 0.6632922887802124, + "score": -0.2295723551876263 + }, + "val": { + "rmse": 0.2384660660716776, + "mae": 0.14107254147529602, + "r2": 0.6201567649841309, + "score": -0.2384660660716776 + }, + "test": { + "rmse": 0.2608031155268875, + "mae": 0.18369406461715698, + "r2": 0.5352010726928711, + "score": -0.2608031155268875 + } + }, + "time": "0:02:19.607379", + "diversity": { + "train": { + "mean_centered_corr": -0.018292076885700226, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0486440509557724 + }, + "val": { + "mean_centered_corr": -0.01807783916592598, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07439792156219482 + }, + "test": { + "mean_centered_corr": -0.018988410010933876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09896504878997803 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25 + ], + "head_selection_time": "0:00:00.015213" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0.toml new file mode 100644 index 000000000..253e6a888 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 4 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.9 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.01 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0/report.json new file mode 100644 index 000000000..23077ce78 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-evaluation/0/report.json @@ -0,0 +1,690 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2176600961009904, + "mae": 0.11412091553211212, + "r2": 0.697328507900238, + "score": -0.2176600961009904 + }, + "val": { + "rmse": 0.23471617503170125, + "mae": 0.12663888931274414, + "r2": 0.6320089101791382, + "score": -0.23471617503170125 + }, + "test": { + "rmse": 0.23405948700639934, + "mae": 0.14594270288944244, + "r2": 0.6256377100944519, + "score": -0.23405948700639934 + } + }, + "time": "0:02:19.607379", + "diversity": { + "train": { + "mean_centered_corr": -0.018292076885700226, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0486440509557724 + }, + "val": { + "mean_centered_corr": -0.01807783916592598, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07439792156219482 + }, + "test": { + "mean_centered_corr": -0.018988410010933876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09896504878997803 + } + }, + "chunk_size": null, + "eval_batch_size": 4096 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..531579d5d --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p9_l0p01-greedy-heads-evaluation/0/report.json @@ -0,0 +1,702 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 4, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 5205992, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.9, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.9005101919174194, + "min_features_per_member": 338.0, + "mean_features_per_member": 353.0, + "max_features_per_member": 364.0, + "min_members_per_feature": 23.0, + "mean_members_per_feature": 28.816326141357422, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.8189943666728325, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.01, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.R", + "shape": [ + 32, + 10415, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.R", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.S", + "shape": [ + 32, + 256, + 4 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2194825831190966, + "mae": 0.114332415163517, + "r2": 0.6922386884689331, + "score": -0.2194825831190966 + }, + "val": { + "rmse": 0.23344479982230373, + "mae": 0.12650029361248016, + "r2": 0.6359846591949463, + "score": -0.23344479982230373 + }, + "test": { + "rmse": 0.2326324214479151, + "mae": 0.14232482016086578, + "r2": 0.6301887631416321, + "score": -0.2326324214479151 + } + }, + "time": "0:02:19.607379", + "diversity": { + "train": { + "mean_centered_corr": -0.018292076885700226, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0486440509557724 + }, + "val": { + "mean_centered_corr": -0.01807783916592598, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07439792156219482 + }, + "test": { + "mean_centered_corr": -0.018988410010933876, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09896504878997803 + } + }, + "chunk_size": null, + "eval_batch_size": 4096, + "heads": [ + 25, + 7, + 22, + 10, + 26, + 16, + 29, + 17, + 19 + ], + "head_selection_time": "0:00:00.146951" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..ad435cbed --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24039030054595545, + "mae": 0.1237230896949768, + "r2": 0.6308118104934692, + "score": -0.24039030054595545 + }, + "val": { + "rmse": 0.23560357747729638, + "mae": 0.13029368221759796, + "r2": 0.6292210817337036, + "score": -0.23560357747729638 + }, + "test": { + "rmse": 0.2389191702722735, + "mae": 0.15265432000160217, + "r2": 0.6099308729171753, + "score": -0.2389191702722735 + } + }, + "time": "0:01:11.607187", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015377" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/1/report.json new file mode 100644 index 000000000..df7faffc7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/1/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9749681353569031, + "min_features_per_member": 376.0, + "mean_features_per_member": 382.1875, + "max_features_per_member": 388.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.1989803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9511310051056044, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.2171563784853777, + "mae": 0.11889506876468658, + "r2": 0.6987277865409851, + "score": -0.2171563784853777 + }, + "val": { + "rmse": 0.24254141605050564, + "mae": 0.13972081243991852, + "r2": 0.607062816619873, + "score": -0.24254141605050564 + }, + "test": { + "rmse": 0.24204913155172003, + "mae": 0.15291987359523773, + "r2": 0.5996437072753906, + "score": -0.24204913155172003 + } + }, + "time": "0:01:33.897406", + "diversity": { + "train": { + "mean_centered_corr": -0.017405925318598747, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049276966601610184 + }, + "val": { + "mean_centered_corr": -0.017822835594415665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07913164794445038 + }, + "test": { + "mean_centered_corr": -0.01794285699725151, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10268954932689667 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27 + ], + "head_selection_time": "0:00:00.015220" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/2/report.json new file mode 100644 index 000000000..996afe498 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/2/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9760841727256775, + "min_features_per_member": 372.0, + "mean_features_per_member": 382.625, + "max_features_per_member": 390.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.23469352722168, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9532826281104112, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2229962847735884, + "mae": 0.12252119928598404, + "r2": 0.6823059320449829, + "score": -0.2229962847735884 + }, + "val": { + "rmse": 0.23847619665119374, + "mae": 0.134144127368927, + "r2": 0.6201244592666626, + "score": -0.23847619665119374 + }, + "test": { + "rmse": 0.2542706881867437, + "mae": 0.17485685646533966, + "r2": 0.5581934452056885, + "score": -0.2542706881867437 + } + }, + "time": "0:01:20.981125", + "diversity": { + "train": { + "mean_centered_corr": -0.018493393436074257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04774053022265434 + }, + "val": { + "mean_centered_corr": -0.018883541226387024, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07628075778484344 + }, + "test": { + "mean_centered_corr": -0.01829403080046177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09460576623678207 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.015828" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml new file mode 100644 index 000000000..3417f3906 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..578c23e0a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23278188472143527, + "mae": 0.12037675827741623, + "r2": 0.6538118124008179, + "score": -0.23278188472143527 + }, + "val": { + "rmse": 0.235050071622046, + "mae": 0.12535566091537476, + "r2": 0.6309611797332764, + "score": -0.235050071622046 + }, + "test": { + "rmse": 0.23184680318718037, + "mae": 0.14313000440597534, + "r2": 0.6326823234558105, + "score": -0.23184680318718037 + } + }, + "time": "0:01:11.607187", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1.toml new file mode 100644 index 000000000..71d78a574 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 1 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1/report.json new file mode 100644 index 000000000..9c63cb100 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/1/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9749681353569031, + "min_features_per_member": 376.0, + "mean_features_per_member": 382.1875, + "max_features_per_member": 388.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.1989803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9511310051056044, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20665772048240127, + "mae": 0.11000452935695648, + "r2": 0.7271542549133301, + "score": -0.20665772048240127 + }, + "val": { + "rmse": 0.23618618194018487, + "mae": 0.12565438449382782, + "r2": 0.6273850798606873, + "score": -0.23618618194018487 + }, + "test": { + "rmse": 0.23253236276211406, + "mae": 0.1399872750043869, + "r2": 0.6305068731307983, + "score": -0.23253236276211406 + } + }, + "time": "0:01:33.897406", + "diversity": { + "train": { + "mean_centered_corr": -0.017405925318598747, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049276966601610184 + }, + "val": { + "mean_centered_corr": -0.017822835594415665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07913164794445038 + }, + "test": { + "mean_centered_corr": -0.01794285699725151, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10268954932689667 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2.toml new file mode 100644 index 000000000..1cfba881b --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 0.0001 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 2 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2/report.json new file mode 100644 index 000000000..253e9fb6a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/2/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9760841727256775, + "min_features_per_member": 372.0, + "mean_features_per_member": 382.625, + "max_features_per_member": 390.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.23469352722168, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9532826281104112, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21767157152228817, + "mae": 0.11745961010456085, + "r2": 0.6972966194152832, + "score": -0.21767157152228817 + }, + "val": { + "rmse": 0.23592642069691294, + "mae": 0.12567678093910217, + "r2": 0.6282042264938354, + "score": -0.23592642069691294 + }, + "test": { + "rmse": 0.2326191697888986, + "mae": 0.14303357899188995, + "r2": 0.6302309036254883, + "score": -0.2326191697888986 + } + }, + "time": "0:01:20.981125", + "diversity": { + "train": { + "mean_centered_corr": -0.018493393436074257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04774053022265434 + }, + "val": { + "mean_centered_corr": -0.018883541226387024, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07628075778484344 + }, + "test": { + "mean_centered_corr": -0.01829403080046177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09460576623678207 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..31384f64a --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,692 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23573796221874224, + "mae": 0.12117554992437363, + "r2": 0.6449635028839111, + "score": -0.23573796221874224 + }, + "val": { + "rmse": 0.2324412121462238, + "mae": 0.12522703409194946, + "r2": 0.6391078233718872, + "score": -0.2324412121462238 + }, + "test": { + "rmse": 0.23820740696857753, + "mae": 0.1556985229253769, + "r2": 0.6122515201568604, + "score": -0.23820740696857753 + } + }, + "time": "0:01:11.607187", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 1, + 25, + 21 + ], + "head_selection_time": "0:00:00.087943" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..645dd9e30 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/1/report.json @@ -0,0 +1,703 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9749681353569031, + "min_features_per_member": 376.0, + "mean_features_per_member": 382.1875, + "max_features_per_member": 388.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.1989803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9511310051056044, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20580531574085753, + "mae": 0.11071916669607162, + "r2": 0.7294004559516907, + "score": -0.20580531574085753 + }, + "val": { + "rmse": 0.23397077829789442, + "mae": 0.12587550282478333, + "r2": 0.6343424916267395, + "score": -0.23397077829789442 + }, + "test": { + "rmse": 0.23276530465893142, + "mae": 0.13940665125846863, + "r2": 0.6297661662101746, + "score": -0.23276530465893142 + } + }, + "time": "0:01:33.897406", + "diversity": { + "train": { + "mean_centered_corr": -0.017405925318598747, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049276966601610184 + }, + "val": { + "mean_centered_corr": -0.017822835594415665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07913164794445038 + }, + "test": { + "mean_centered_corr": -0.01794285699725151, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10268954932689667 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27, + 6, + 1, + 15, + 2, + 13, + 0, + 6, + 1, + 29, + 31, + 5, + 1, + 21, + 6, + 22 + ], + "head_selection_time": "0:00:00.252986" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..1ecac8196 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-greedy-heads-evaluation/2/report.json @@ -0,0 +1,696 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 0.0001, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9760841727256775, + "min_features_per_member": 372.0, + "mean_features_per_member": 382.625, + "max_features_per_member": 390.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.23469352722168, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9532826281104112, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2210309033660134, + "mae": 0.12036030739545822, + "r2": 0.6878812313079834, + "score": -0.2210309033660134 + }, + "val": { + "rmse": 0.23411027740292373, + "mae": 0.128285214304924, + "r2": 0.633906364440918, + "score": -0.23411027740292373 + }, + "test": { + "rmse": 0.24496736545125247, + "mae": 0.16705775260925293, + "r2": 0.589931845664978, + "score": -0.24496736545125247 + } + }, + "time": "0:01:20.981125", + "diversity": { + "train": { + "mean_centered_corr": -0.018493393436074257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04774053022265434 + }, + "val": { + "mean_centered_corr": -0.018883541226387024, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07628075778484344 + }, + "test": { + "mean_centered_corr": -0.01829403080046177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09460576623678207 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24, + 4, + 27, + 6, + 3, + 30, + 5, + 27, + 4 + ], + "head_selection_time": "0:00:00.147559" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json new file mode 100644 index 000000000..022c71440 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.24039030054595545, + "mae": 0.1237230896949768, + "r2": 0.6308118104934692, + "score": -0.24039030054595545 + }, + "val": { + "rmse": 0.23560357747729638, + "mae": 0.13029368221759796, + "r2": 0.6292210817337036, + "score": -0.23560357747729638 + }, + "test": { + "rmse": 0.2389191702722735, + "mae": 0.15265432000160217, + "r2": 0.6099308729171753, + "score": -0.2389191702722735 + } + }, + "time": "0:01:13.814362", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4 + ], + "head_selection_time": "0:00:00.015454" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/1/report.json new file mode 100644 index 000000000..051fe52d7 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/1/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9749681353569031, + "min_features_per_member": 376.0, + "mean_features_per_member": 382.1875, + "max_features_per_member": 388.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.1989803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9511310051056044, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.2171563784853777, + "mae": 0.11889506876468658, + "r2": 0.6987277865409851, + "score": -0.2171563784853777 + }, + "val": { + "rmse": 0.24254141605050564, + "mae": 0.13972081243991852, + "r2": 0.607062816619873, + "score": -0.24254141605050564 + }, + "test": { + "rmse": 0.24204913155172003, + "mae": 0.15291987359523773, + "r2": 0.5996437072753906, + "score": -0.24204913155172003 + } + }, + "time": "0:01:33.744311", + "diversity": { + "train": { + "mean_centered_corr": -0.017405925318598747, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049276966601610184 + }, + "val": { + "mean_centered_corr": -0.017822835594415665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07913164794445038 + }, + "test": { + "mean_centered_corr": -0.01794285699725151, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10268954932689667 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27 + ], + "head_selection_time": "0:00:00.015010" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/2/report.json new file mode 100644 index 000000000..bc47fc492 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/2/report.json @@ -0,0 +1,688 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9760841727256775, + "min_features_per_member": 372.0, + "mean_features_per_member": 382.625, + "max_features_per_member": 390.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.23469352722168, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9532826281104112, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2229962847735884, + "mae": 0.12252119928598404, + "r2": 0.6823059320449829, + "score": -0.2229962847735884 + }, + "val": { + "rmse": 0.23847619665119374, + "mae": 0.134144127368927, + "r2": 0.6201244592666626, + "score": -0.23847619665119374 + }, + "test": { + "rmse": 0.2542706881867437, + "mae": 0.17485685646533966, + "r2": 0.5581934452056885, + "score": -0.2542706881867437 + } + }, + "time": "0:01:19.517367", + "diversity": { + "train": { + "mean_centered_corr": -0.018493393436074257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04774053022265434 + }, + "val": { + "mean_centered_corr": -0.018883541226387024, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07628075778484344 + }, + "test": { + "mean_centered_corr": -0.01829403080046177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09460576623678207 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24 + ], + "head_selection_time": "0:00:00.015325" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml new file mode 100644 index 000000000..aff429e28 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 0 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 0 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/report.json new file mode 100644 index 000000000..193998f2e --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23278188472143527, + "mae": 0.12037675827741623, + "r2": 0.6538118124008179, + "score": -0.23278188472143527 + }, + "val": { + "rmse": 0.235050071622046, + "mae": 0.12535566091537476, + "r2": 0.6309611797332764, + "score": -0.235050071622046 + }, + "test": { + "rmse": 0.23184680318718037, + "mae": 0.14313000440597534, + "r2": 0.6326823234558105, + "score": -0.23184680318718037 + } + }, + "time": "0:01:13.814362", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1.toml new file mode 100644 index 000000000..ad01c71b3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 1 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 1 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1/report.json new file mode 100644 index 000000000..f4c5d18c3 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/1/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9749681353569031, + "min_features_per_member": 376.0, + "mean_features_per_member": 382.1875, + "max_features_per_member": 388.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.1989803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9511310051056044, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20665772048240127, + "mae": 0.11000452935695648, + "r2": 0.7271542549133301, + "score": -0.20665772048240127 + }, + "val": { + "rmse": 0.23618618194018487, + "mae": 0.12565438449382782, + "r2": 0.6273850798606873, + "score": -0.23618618194018487 + }, + "test": { + "rmse": 0.23253236276211406, + "mae": 0.1399872750043869, + "r2": 0.6305068731307983, + "score": -0.23253236276211406 + } + }, + "time": "0:01:33.744311", + "diversity": { + "train": { + "mean_centered_corr": -0.017405925318598747, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049276966601610184 + }, + "val": { + "mean_centered_corr": -0.017822835594415665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07913164794445038 + }, + "test": { + "mean_centered_corr": -0.01794285699725151, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10268954932689667 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2.toml b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2.toml new file mode 100644 index 000000000..86addbb91 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2.toml @@ -0,0 +1,65 @@ +batch_size = 256 +patience = 16 +n_epochs = -1 +gradient_clipping_norm = 1.0 +amp = true +seed = 2 +head_selection = true +rla_adapter_lr_multiplier = 1.0 +rla_extra_paths_freeze_fraction = 0.0 + +[data] +cache = true +path = "data/sberbank-housing" +num_policy = "noisy-quantile" +cat_policy = "ordinal" + +[optimizer] +lr = 0.0008922700423431547 +weight_decay = 0.00021333759467820313 + +[model] +arch_type = "tabm" +k = 32 +share_training_batches = false +rla_rank = 1 +rla_first_only = false +rla_additive = false +rla_init = "base_preserving" +rla_base_preserve_noise = 1e-05 + +[model.backbone] +n_blocks = 3 +d_block = 256 +dropout = 0.3128700072370906 + +[model.num_embeddings] +type = "PiecewiseLinearEmbeddings" +d_embedding = 28 + +[model.mfb] +enabled = true +mask_mode = "member_fixed" +mask_granularity = "feature_group" +keep_rate = 0.975 +training_only = false +inverted_scaling = true +use_soft_mask = false +mask_strength = 1.0 +anchor_fraction = 0.0 +warmup_epochs = 0 +mask_seed = 2 + +[bins] +n_bins = 84 + +[cf_fisd] +lambda = 0.001 +variant = "raw" +dataset_name = "sberbank-housing" +teacher_dir = "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing" +teacher_names = [ + "xgb", + "lgbm", + "cat", +] diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2/report.json new file mode 100644 index 000000000..65d6d6497 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/2/report.json @@ -0,0 +1,684 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9760841727256775, + "min_features_per_member": 372.0, + "mean_features_per_member": 382.625, + "max_features_per_member": 390.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.23469352722168, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9532826281104112, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.21767157152228817, + "mae": 0.11745961010456085, + "r2": 0.6972966194152832, + "score": -0.21767157152228817 + }, + "val": { + "rmse": 0.23592642069691294, + "mae": 0.12567678093910217, + "r2": 0.6282042264938354, + "score": -0.23592642069691294 + }, + "test": { + "rmse": 0.2326191697888986, + "mae": 0.14303357899188995, + "r2": 0.6302309036254883, + "score": -0.2326191697888986 + } + }, + "time": "0:01:19.517367", + "diversity": { + "train": { + "mean_centered_corr": -0.018493393436074257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04774053022265434 + }, + "val": { + "mean_centered_corr": -0.018883541226387024, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07628075778484344 + }, + "test": { + "mean_centered_corr": -0.01829403080046177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09460576623678207 + } + }, + "chunk_size": null, + "eval_batch_size": 32768 +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json new file mode 100644 index 000000000..55f69c783 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/0/report.json @@ -0,0 +1,692 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 0, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 0 + }, + "mask_stats": { + "keep_rate_actual": 0.976961076259613, + "min_features_per_member": 375.0, + "mean_features_per_member": 382.96875, + "max_features_per_member": 391.0, + "min_members_per_feature": 27.0, + "mean_members_per_feature": 31.262754440307617, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9552194924522834, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 666, + "metrics": { + "train": { + "rmse": 0.23573796221874224, + "mae": 0.12117554992437363, + "r2": 0.6449635028839111, + "score": -0.23573796221874224 + }, + "val": { + "rmse": 0.2324412121462238, + "mae": 0.12522703409194946, + "r2": 0.6391078233718872, + "score": -0.2324412121462238 + }, + "test": { + "rmse": 0.23820740696857753, + "mae": 0.1556985229253769, + "r2": 0.6122515201568604, + "score": -0.23820740696857753 + } + }, + "time": "0:01:13.814362", + "diversity": { + "train": { + "mean_centered_corr": -0.018871087580919266, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04625045508146286 + }, + "val": { + "mean_centered_corr": -0.019568709656596184, + "mean_pairwise_disagreement": NaN, + "member_std": 0.0614706352353096 + }, + "test": { + "mean_centered_corr": -0.018940167501568794, + "mean_pairwise_disagreement": NaN, + "member_std": 0.08073103427886963 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 4, + 0, + 1, + 25, + 21 + ], + "head_selection_time": "0:00:00.088240" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/1/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/1/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/1/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/1/report.json new file mode 100644 index 000000000..9dcd68238 --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/1/report.json @@ -0,0 +1,703 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 1, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 1 + }, + "mask_stats": { + "keep_rate_actual": 0.9749681353569031, + "min_features_per_member": 376.0, + "mean_features_per_member": 382.1875, + "max_features_per_member": 388.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.1989803314209, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9511310051056044, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 1184, + "metrics": { + "train": { + "rmse": 0.20580531574085753, + "mae": 0.11071916669607162, + "r2": 0.7294004559516907, + "score": -0.20580531574085753 + }, + "val": { + "rmse": 0.23397077829789442, + "mae": 0.12587550282478333, + "r2": 0.6343424916267395, + "score": -0.23397077829789442 + }, + "test": { + "rmse": 0.23276530465893142, + "mae": 0.13940665125846863, + "r2": 0.6297661662101746, + "score": -0.23276530465893142 + } + }, + "time": "0:01:33.744311", + "diversity": { + "train": { + "mean_centered_corr": -0.017405925318598747, + "mean_pairwise_disagreement": NaN, + "member_std": 0.049276966601610184 + }, + "val": { + "mean_centered_corr": -0.017822835594415665, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07913164794445038 + }, + "test": { + "mean_centered_corr": -0.01794285699725151, + "mean_pairwise_disagreement": NaN, + "member_std": 0.10268954932689667 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 27, + 6, + 1, + 15, + 2, + 13, + 0, + 6, + 1, + 29, + 31, + 5, + 1, + 21, + 6, + 22 + ], + "head_selection_time": "0:00:00.247440" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/2/DONE b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/2/DONE new file mode 100644 index 000000000..e69de29bb diff --git a/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/2/report.json b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/2/report.json new file mode 100644 index 000000000..88ac1a9ce --- /dev/null +++ b/tabm_integrated/paper/exp/integrated/rescue_sberbank_combined_confirm/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-greedy-heads-evaluation/2/report.json @@ -0,0 +1,696 @@ +{ + "function": "bin.run_integrated.main", + "gpus": [ + "NVIDIA GeForce RTX 4090" + ], + "config": { + "batch_size": 256, + "patience": 16, + "n_epochs": -1, + "gradient_clipping_norm": 1.0, + "amp": true, + "seed": 2, + "head_selection": true, + "rla_adapter_lr_multiplier": 1.0, + "rla_extra_paths_freeze_fraction": 0.0, + "data": { + "cache": true, + "path": "data/sberbank-housing", + "num_policy": "noisy-quantile", + "cat_policy": "ordinal" + }, + "optimizer": { + "lr": 0.0008922700423431547, + "weight_decay": 0.00021333759467820313 + }, + "model": { + "arch_type": "tabm", + "k": 32, + "share_training_batches": false, + "rla_rank": 1, + "rla_first_only": false, + "rla_additive": false, + "rla_init": "base_preserving", + "rla_base_preserve_noise": 1e-05, + "backbone": { + "n_blocks": 3, + "d_block": 256, + "dropout": 0.3128700072370906 + }, + "num_embeddings": { + "type": "PiecewiseLinearEmbeddings", + "d_embedding": 28 + }, + "mfb": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + } + }, + "bins": { + "n_bins": 84 + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "dataset_name": "sberbank-housing", + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ] + } + }, + "n_parameters": 4083272, + "prediction_type": "labels", + "mfb": { + "enabled": true, + "config": { + "enabled": true, + "mask_mode": "member_fixed", + "mask_granularity": "feature_group", + "keep_rate": 0.975, + "training_only": false, + "inverted_scaling": true, + "use_soft_mask": false, + "mask_strength": 1.0, + "anchor_fraction": 0.0, + "warmup_epochs": 0, + "mask_seed": 2 + }, + "mask_stats": { + "keep_rate_actual": 0.9760841727256775, + "min_features_per_member": 372.0, + "mean_features_per_member": 382.625, + "max_features_per_member": 390.0, + "min_members_per_feature": 28.0, + "mean_members_per_feature": 31.23469352722168, + "max_members_per_feature": 32.0, + "mean_pairwise_jaccard": 0.9532826281104112, + "anchor_fraction_configured": 0.0, + "n_anchor_members": 0.0, + "core_fraction_configured": 0.0, + "n_core_features": 0.0 + } + }, + "cf_fisd": { + "lambda": 0.001, + "variant": "raw", + "teacher_names": [ + "xgb", + "lgbm", + "cat" + ], + "member_groups": { + "xgb": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "lgbm": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "cat": [ + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + "d_features": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 24, + 27, + 18, + 35, + 8, + 14, + 5, + 6, + 6, + 18, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_features": 392, + "teacher_dir": "../../cf_fisd_recovered/paper/exp/cf_fisd/_teachers/tabred/sberbank-housing", + "start_epoch": 0, + "mode": "raw" + }, + "epoch_size": 74, + "ncl": { + "use_ncl": false, + "lambda_ncl": 0.0, + "ncl_warmup_epochs": 0, + "ncl_space": "logits" + }, + "esam": { + "use_esam": false, + "esam_rho": 0.01, + "esam_eps": 1e-12, + "esam_adapter_only": true, + "esam_memberwise": true, + "esam_warmup_epochs": 0, + "esam_start_epoch": 0, + "esam_end_epoch": -1, + "esam_log_diagnostics": true, + "esam_diagnostics_every": 100, + "adapter_parameter_names": [ + { + "name": "backbone.blocks.0.0.r", + "shape": [ + 32, + 10415 + ] + }, + { + "name": "backbone.blocks.0.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.0.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.1.0.bias", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.r", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.s", + "shape": [ + 32, + 256 + ] + }, + { + "name": "backbone.blocks.2.0.bias", + "shape": [ + 32, + 256 + ] + } + ], + "diagnostics_path": null, + "n_diagnostics_records": 0 + }, + "rla_training": { + "adapter_lr_multiplier": 1.0, + "extra_paths_freeze_fraction": 0.0, + "extra_paths_freeze_until_epoch": 0 + }, + "best_step": 888, + "metrics": { + "train": { + "rmse": 0.2210309033660134, + "mae": 0.12036030739545822, + "r2": 0.6878812313079834, + "score": -0.2210309033660134 + }, + "val": { + "rmse": 0.23411027740292373, + "mae": 0.128285214304924, + "r2": 0.633906364440918, + "score": -0.23411027740292373 + }, + "test": { + "rmse": 0.24496736545125247, + "mae": 0.16705775260925293, + "r2": 0.589931845664978, + "score": -0.24496736545125247 + } + }, + "time": "0:01:19.517367", + "diversity": { + "train": { + "mean_centered_corr": -0.018493393436074257, + "mean_pairwise_disagreement": NaN, + "member_std": 0.04774053022265434 + }, + "val": { + "mean_centered_corr": -0.018883541226387024, + "mean_pairwise_disagreement": NaN, + "member_std": 0.07628075778484344 + }, + "test": { + "mean_centered_corr": -0.01829403080046177, + "mean_pairwise_disagreement": NaN, + "member_std": 0.09460576623678207 + } + }, + "chunk_size": null, + "eval_batch_size": 32768, + "heads": [ + 24, + 4, + 27, + 6, + 3, + 30, + 5, + 27, + 4 + ], + "head_selection_time": "0:00:00.145674" +} \ No newline at end of file diff --git a/tabm_integrated/paper/exp/sberbank_combined_rescue_confirmed.csv b/tabm_integrated/paper/exp/sberbank_combined_rescue_confirmed.csv new file mode 100644 index 000000000..6ac8242a1 --- /dev/null +++ b/tabm_integrated/paper/exp/sberbank_combined_rescue_confirmed.csv @@ -0,0 +1,3 @@ +wave,dataset,variant,seed,metric,metric_key,direction,validation_metric,validation_score,test_metric,inference_mode,config_path,result_path,failure,best_step,matched_validation_baseline,validation_delta,validation_percent_delta,validation_status,mean,std,baseline_mean,baseline_std,absolute_delta,percent_delta,status +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.23560357747729638,-0.23560357747729638,0.2389191702722735,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0,False,666,0.23955454428788905,0.0039509668105926765,1.6492973749830142,validation_win,0.24507966333691242,0.008112050773101143,0.2502002926581795,0.013600655537586393,0.005120629321267084,2.0466120430413817,weak_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.23560357747729638,-0.23560357747729638,0.2389191702722735,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0,False,666,0.23955454428788905,0.0039509668105926765,1.6492973749830142,validation_win,0.24507966333691242,0.008112050773101143,0.2502002926581795,0.013600655537586393,0.005120629321267084,2.0466120430413817,weak_win diff --git a/tabm_integrated/paper/exp/sberbank_combined_rescue_selected.csv b/tabm_integrated/paper/exp/sberbank_combined_rescue_selected.csv new file mode 100644 index 000000000..bfc7a42a6 --- /dev/null +++ b/tabm_integrated/paper/exp/sberbank_combined_rescue_selected.csv @@ -0,0 +1,21 @@ +wave,dataset,variant,seed,metric,metric_key,direction,validation_metric,validation_score,test_metric,inference_mode,config_path,result_path,failure,best_step,matched_validation_baseline,validation_delta,validation_percent_delta,validation_status +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.23560357747729638,-0.23560357747729638,0.2389191702722735,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-best-head-evaluation/0,False,666,0.23955454428788905,0.0039509668105926765,1.6492973749830142,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.23560357747729638,-0.23560357747729638,0.2389191702722735,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p001-best-head-evaluation/0,False,666,0.23955454428788905,0.0039509668105926765,1.6492973749830142,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001,0,RMSE,rmse,lower,0.23560357747729638,-0.23560357747729638,0.2389191702722735,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p001-best-head-evaluation/0,False,666,0.23955454428788905,0.0039509668105926765,1.6492973749830142,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001,0,RMSE,rmse,lower,0.2357278561917395,-0.2357278561917395,0.24177902137278479,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p001-best-head-evaluation/0,False,666,0.23955454428788905,0.0038266880961495553,1.5974182863134347,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005,0,RMSE,rmse,lower,0.2360115531890879,-0.2360115531890879,0.23473422019733492,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p005-best-head-evaluation/0,False,666,0.23955454428788905,0.003542991098801157,1.4789913960234888,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001,0,RMSE,rmse,lower,0.23604388522704514,-0.23604388522704514,0.2413116661749965,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p001-best-head-evaluation/0,False,666,0.23955454428788905,0.0035106590608439125,1.4654946627207013,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005,0,RMSE,rmse,lower,0.23613982935624828,-0.23613982935624828,0.23869431760070778,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p95_l0p005-best-head-evaluation/0,False,666,0.23955454428788905,0.0034147149316407777,1.4254436048339294,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005,0,RMSE,rmse,lower,0.2361938000242818,-0.2361938000242818,0.23968299921179284,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p005-best-head-evaluation/0,False,666,0.23955454428788905,0.003360744263607246,1.4029140100838204,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01,0,RMSE,rmse,lower,0.23633714926648194,-0.23633714926648194,0.23801691214143877,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p95_l0p01-best-head-evaluation/0,False,666,0.23955454428788905,0.00321739502140711,1.3430740923622584,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.23637143045682502,-0.23637143045682502,0.2444339973181191,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n0p0001_k0p975_l0p01-best-head-evaluation/0,False,666,0.23955454428788905,0.0031831138310640383,1.3287637020313308,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.236445130001989,-0.236445130001989,0.2384320235467145,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p0001_k0p975_l0p01-best-head-evaluation/0,False,666,0.23955454428788905,0.0031094142859000584,1.2979984559020776,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01,0,RMSE,rmse,lower,0.23656618722639788,-0.23656618722639788,0.2443526301182123,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p01-best-head-evaluation/0,False,666,0.23955454428788905,0.0029883570614911703,1.2474641507530149,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01,0,RMSE,rmse,lower,0.23656618722639788,-0.23656618722639788,0.2443526301182123,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p0001_k0p975_l0p01-best-head-evaluation/0,False,666,0.23955454428788905,0.0029883570614911703,1.2474641507530149,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01,0,RMSE,rmse,lower,0.23656618722639788,-0.23656618722639788,0.2443526301182123,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n0p001_k0p975_l0p01-best-head-evaluation/0,False,666,0.23955454428788905,0.0029883570614911703,1.2474641507530149,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005,0,RMSE,rmse,lower,0.23674080812018197,-0.23674080812018197,0.24022462096222602,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p975_l0p005-best-head-evaluation/0,False,666,0.23955454428788905,0.0028137361677070816,1.1745701489701748,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r4_n0p001_rho0p0005,0,RMSE,rmse,lower,0.23686825686043878,-0.23686825686043878,0.24350702848855615,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r4_n0p001_rho0p0005-best-head-evaluation/0,False,666,0.23955454428788905,0.002686287427450279,1.1213677600797187,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005,0,RMSE,rmse,lower,0.2369960693297996,-0.2369960693297996,0.2449247661219165,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r2_n1em05_k0p975_l0p005-best-head-evaluation/0,False,666,0.23955454428788905,0.002558474958089446,1.0680135355790839,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001,0,RMSE,rmse,lower,0.237046057654127,-0.237046057654127,0.2530936074511429,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n0p001_k0p9_l0p001-best-head-evaluation/0,False,888,0.23955454428788905,0.002508486633762047,1.0471463362212103,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_esam_r2_n0p0001_rho0p00025,0,RMSE,rmse,lower,0.23704860354665752,-0.23704860354665752,0.23874426241915847,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_esam_r2_n0p0001_rho0p00025-best-head-evaluation/0,False,814,0.23955454428788905,0.0025059407412315327,1.046083575112636,validation_win +rescue_sberbank_combined,sberbank-housing,sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005,0,RMSE,rmse,lower,0.23707190033649148,-0.23707190033649148,0.2397359546762736,best-head,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-evaluation/0.toml,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r4_n1em05_k0p95_l0p005-best-head-evaluation/0,False,666,0.23955454428788905,0.0024826439513975784,1.0363585290262813,validation_win diff --git a/tabm_integrated/paper/exp/selected_integrated_configs.csv b/tabm_integrated/paper/exp/selected_integrated_configs.csv index 393b807a6..c60733aac 100644 --- a/tabm_integrated/paper/exp/selected_integrated_configs.csv +++ b/tabm_integrated/paper/exp/selected_integrated_configs.csv @@ -4,7 +4,7 @@ sberbank-housing,best_rla_only,rla_rank2_noise0.001,sweeps,greedy-heads,0.232861 sberbank-housing,best_esam_only,esam_rho0.0025,sweeps,greedy-heads,0.23295802298295287,-0.23295802298295287,exp/integrated/sweeps/sberbank-housing/esam_rho0.0025-evaluation/0.toml,,,, sberbank-housing,best_mfb_only,mfb_keep0.8,sweeps,greedy-heads,0.23237596587359466,-0.23237596587359466,exp/integrated/sweeps/sberbank-housing/mfb_keep0.8-evaluation/0.toml,,,, sberbank-housing,best_cf_fisd_only,cf_fisd_only,smoke,greedy-heads,0.23327114512588593,-0.23327114512588593,exp/integrated/smoke/sberbank-housing/cf_fisd_only-evaluation/0.toml,,,, -sberbank-housing,best_combined,mfb_cf_fisd,sweeps,greedy-heads,0.23320900639668657,-0.23320900639668657,exp/integrated/sweeps/sberbank-housing/mfb_cf_fisd-evaluation/0.toml,,,, +sberbank-housing,best_combined,sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001,rescue_sberbank_combined,best-head,0.23560357747729638,-0.23560357747729638,exp/integrated/rescue_sberbank_combined/sberbank-housing/sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001-evaluation/0.toml,0.23955454428788905,0.0039509668105926765,1.6492973749830142,validation_win ecom-offers,baseline_plr,baseline_plr,smoke,mean,,,exp/integrated/smoke/ecom-offers/baseline_plr-evaluation/0.toml,,,, ecom-offers,best_rla_only,rla_rank2_noise0.0001,sweeps,best-head,0.6396541901648656,0.6396541901648656,exp/integrated/sweeps/ecom-offers/rla_rank2_noise0.0001-evaluation/0.toml,,,, ecom-offers,best_esam_only,esam_rho0.001,sweeps,greedy-heads,0.641845068859757,0.641845068859757,exp/integrated/sweeps/ecom-offers/esam_rho0.001-evaluation/0.toml,,,, diff --git a/tabm_integrated/tools/sberbank_combined_rescue.py b/tabm_integrated/tools/sberbank_combined_rescue.py new file mode 100644 index 000000000..d7c931a94 --- /dev/null +++ b/tabm_integrated/tools/sberbank_combined_rescue.py @@ -0,0 +1,266 @@ +from __future__ import annotations + +import argparse +import csv +import itertools +import json +import math +import shutil +import sys +import tomllib +from pathlib import Path +from typing import Any + +sys.dont_write_bytecode = True + +import tomli_w + +TOOLS = Path(__file__).resolve().parent +PAPER = TOOLS.parent / 'paper' +EXP_ROOT = PAPER / 'exp' / 'integrated' +DATASET = 'sberbank-housing' +WAVE = 'rescue_sberbank_combined' +CONFIRM_WAVE = 'rescue_sberbank_combined_confirm' +MANIFEST = EXP_ROOT / 'manifest_sberbank_combined_rescue_seed0.txt' +CONFIRM_MANIFEST = EXP_ROOT / 'manifest_sberbank_combined_rescue_final.txt' +SELECTED_PATH = PAPER / 'exp' / 'sberbank_combined_rescue_selected.csv' +FINAL_SELECTED = PAPER / 'exp' / 'selected_integrated_configs.csv' + +sys.path.insert(0, str(TOOLS)) +import aggregate_integrated_results as agg # noqa: E402 +import generate_integrated_configs as gen # noqa: E402 + + +def ftag(x: float | int) -> str: + if isinstance(x, int): + return str(x) + if x == 0: + return '0' + return f'{x:g}'.replace('-', 'm').replace('.', 'p') + + +def write_config(wave: str, variant: str, cfg: dict[str, Any], seed: int) -> Path: + cfg = json.loads(json.dumps(cfg)) + cfg['seed'] = int(seed) + if cfg.get('model', {}).get('mfb'): + cfg['model']['mfb']['mask_seed'] = int(seed) + path = EXP_ROOT / wave / DATASET / f'{variant}-evaluation' / f'{seed}.toml' + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(tomli_w.dumps(cfg)) + return path + + +def generate() -> None: + paths: list[Path] = [] + for rank, noise, lam in itertools.product([1, 2, 4], [0.0, 1e-5, 1e-4, 1e-3], [0.001, 0.005, 0.01, 0.02]): + cfg = gen.base_config(DATASET, 0) + gen.apply_rla(cfg, rank=rank, noise=noise) + gen.apply_cf_fisd(cfg, DATASET, lam=lam) + variant = f'sb_comb_rla_cf_r{rank}_n{ftag(noise)}_l{ftag(lam)}' + paths.append(write_config(WAVE, variant, cfg, 0)) + for rank, noise, rho in itertools.product([1, 2, 4], [0.0, 1e-5, 1e-4, 1e-3], [0.00025, 0.0005, 0.001, 0.0025]): + cfg = gen.base_config(DATASET, 0) + gen.apply_rla(cfg, rank=rank, noise=noise) + gen.apply_esam(cfg, rho=rho) + variant = f'sb_comb_rla_esam_r{rank}_n{ftag(noise)}_rho{ftag(rho)}' + paths.append(write_config(WAVE, variant, cfg, 0)) + for rank, noise, keep, lam in itertools.product([1, 2, 4], [1e-5, 1e-4, 1e-3], [0.9, 0.95, 0.975], [0.001, 0.005, 0.01]): + cfg = gen.base_config(DATASET, 0) + gen.apply_rla(cfg, rank=rank, noise=noise) + gen.apply_mfb(cfg, keep=keep) + gen.apply_cf_fisd(cfg, DATASET, lam=lam) + variant = f'sb_comb_rla_mfb_cf_r{rank}_n{ftag(noise)}_k{ftag(keep)}_l{ftag(lam)}' + paths.append(write_config(WAVE, variant, cfg, 0)) + MANIFEST.write_text('\n'.join(str(p.relative_to(PAPER)) for p in paths) + '\n') + print(f'{MANIFEST.relative_to(PAPER)} configs={len(paths)}') + + +def signed_delta(value: float, baseline: float, direction: str) -> tuple[float, float]: + if direction == 'lower': + d = baseline - value + return d, 100.0 * d / baseline if baseline else float('nan') + d = value - baseline + return d, 100.0 * d / abs(baseline) if baseline else float('nan') + + +def collect_candidates(wave: str, seed: int = 0) -> list[dict[str, Any]]: + rows = agg.collect_rows() + baselines = { + r['inference_mode']: r + for r in rows + if r['wave'] == 'final' + and r['dataset'] == DATASET + and r['variant'] == 'baseline_plr' + and r['seed'] == seed + and not r['failure'] + } + missing = [x for x in ['mean', 'best-head', 'greedy-heads'] if x not in baselines] + if missing: + raise RuntimeError(f'missing matched baselines for {missing}') + out: list[dict[str, Any]] = [] + for r in rows: + if r['wave'] != wave or r['dataset'] != DATASET or r['seed'] != seed or r['failure']: + continue + b = baselines[r['inference_mode']] + d, pct = signed_delta(r['validation_metric'], b['validation_metric'], r['direction']) + e = dict(r) + e['matched_validation_baseline'] = b['validation_metric'] + e['validation_delta'] = d + e['validation_percent_delta'] = pct + e['validation_status'] = 'validation_win' if d > 1e-12 else ('tie' if abs(d) <= 1e-12 else 'validation_loss') + out.append(e) + out.sort(key=lambda r: (r['validation_delta'], r['validation_score']), reverse=True) + return out + + +def write_csv(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if not rows: + path.write_text('') + return + fields: list[str] = [] + for row in rows: + for key in row: + if key not in fields: + fields.append(key) + with path.open('w', newline='') as f: + writer = csv.DictWriter(f, fieldnames=fields, lineterminator='\n') + writer.writeheader() + writer.writerows(rows) + + +def select_seed0() -> None: + candidates = collect_candidates(WAVE, 0) + top = candidates[:20] + write_csv(SELECTED_PATH, top) + winners = [r for r in candidates if r['validation_delta'] > 1e-12] + confirm = winners[:2] + paths: list[Path] = [] + for row in confirm: + cfg = tomllib.loads((PAPER / row['config_path']).read_text()) + for seed in [0, 1, 2]: + paths.append(write_config(CONFIRM_WAVE, row['variant'], cfg, seed)) + CONFIRM_MANIFEST.write_text('\n'.join(str(p.relative_to(PAPER)) for p in paths) + ('\n' if paths else '')) + print(f'{SELECTED_PATH.relative_to(PAPER)} top_rows={len(top)} validation_winners={len(winners)}') + print(f'{CONFIRM_MANIFEST.relative_to(PAPER)} configs={len(paths)}') + for row in top[:5]: + print(row['variant'], row['inference_mode'], row['validation_metric'], row['matched_validation_baseline'], row['validation_delta'], row['validation_status']) + + +def test_stats_for_variant(variant: str, inference: str) -> dict[str, Any] | None: + rows = agg.collect_rows() + baselines = [ + r for r in rows + if r['wave'] == 'final' + and r['dataset'] == DATASET + and r['variant'] == 'baseline_plr' + and r['inference_mode'] == inference + and not r['failure'] + ] + rs = [ + r for r in rows + if r['wave'] == CONFIRM_WAVE + and r['dataset'] == DATASET + and r['variant'] == variant + and r['inference_mode'] == inference + and not r['failure'] + ] + if len(rs) != 3 or len(baselines) != 3: + return None + import statistics + rs = sorted(rs, key=lambda r: r['seed']) + baselines = sorted(baselines, key=lambda r: r['seed']) + mean = statistics.mean(r['test_metric'] for r in rs) + std = statistics.stdev(r['test_metric'] for r in rs) + base_mean = statistics.mean(r['test_metric'] for r in baselines) + base_std = statistics.stdev(r['test_metric'] for r in baselines) + delta, pct = signed_delta(mean, base_mean, rs[0]['direction']) + status = agg.status_for(delta, base_std, 3, False) + return { + 'variant': variant, + 'inference_mode': inference, + 'mean': mean, + 'std': std, + 'baseline_mean': base_mean, + 'baseline_std': base_std, + 'absolute_delta': delta, + 'percent_delta': pct, + 'status': status, + } + + +def update_final_if_win() -> None: + selected = [] + if SELECTED_PATH.exists() and SELECTED_PATH.read_text().strip(): + with SELECTED_PATH.open() as f: + for row in csv.DictReader(f): + if float(row['validation_delta']) > 1e-12: + selected.append(row) + selected = selected[:2] + results = [] + chosen = None + for row in selected: + stats = test_stats_for_variant(row['variant'], row['inference_mode']) + if stats is None: + continue + result = dict(row) | stats + results.append(result) + if chosen is None and stats['status'] in {'clear_win', 'weak_win'}: + chosen = result + write_csv(PAPER / 'exp' / 'sberbank_combined_rescue_confirmed.csv', results) + if chosen is None: + print('no_confirmed_win') + for r in results: + print(r['variant'], r['inference_mode'], r['absolute_delta'], r['status']) + return + + variant = chosen['variant'] + # Copy confirmation outputs into final best_combined slots. + for seed in [0, 1, 2]: + src_cfg = EXP_ROOT / CONFIRM_WAVE / DATASET / f'{variant}-evaluation' / f'{seed}.toml' + dst_cfg = EXP_ROOT / 'final' / DATASET / 'best_combined-evaluation' / f'{seed}.toml' + dst_cfg.write_text(src_cfg.read_text()) + for suffix in ['-evaluation', '-best-head-evaluation', '-greedy-heads-evaluation']: + src = EXP_ROOT / CONFIRM_WAVE / DATASET / f'{variant}{suffix}' / str(seed) + dst = EXP_ROOT / 'final' / DATASET / f'best_combined{suffix}' / str(seed) + if src.exists(): + if dst.exists(): + shutil.rmtree(dst) + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(src, dst) + # Update validation-selected configs. + rows = [] + with FINAL_SELECTED.open() as f: + for row in csv.DictReader(f): + if row['dataset'] == DATASET and row['final_variant'] == 'best_combined': + row.update({ + 'source_variant': chosen['variant'], + 'source_wave': WAVE, + 'inference_mode': chosen['inference_mode'], + 'validation_metric': chosen['validation_metric'], + 'validation_score': chosen['validation_score'], + 'matched_validation_baseline': str(chosen['matched_validation_baseline']), + 'validation_delta': str(chosen['validation_delta']), + 'validation_percent_delta': str(chosen['validation_percent_delta']), + 'validation_status': chosen['validation_status'], + 'source_config_path': chosen['config_path'], + }) + rows.append(row) + write_csv(FINAL_SELECTED, rows) + print('updated_final_best_combined', chosen['variant'], chosen['inference_mode'], chosen['absolute_delta'], chosen['status']) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('stage', choices=['generate', 'select-seed0', 'update-final-if-win']) + args = parser.parse_args() + if args.stage == 'generate': + generate() + elif args.stage == 'select-seed0': + select_seed0() + elif args.stage == 'update-final-if-win': + update_final_if_win() + + +if __name__ == '__main__': + main() From 19659adedea71746247193b1f9a031cafa01128a Mon Sep 17 00:00:00 2001 From: abhipray-chavan Date: Mon, 27 Apr 2026 10:47:03 +0800 Subject: [PATCH 6/7] Add final report --- final_report/.gitignore | 2 + final_report/aaai2026.bst | 1493 +++++++++++++++++ final_report/aaai2026.sty | 318 ++++ final_report/build_report.py | 777 +++++++++ final_report/fig_combined_comparison.pdf | Bin 0 -> 14951 bytes final_report/fig_combined_comparison.png | Bin 0 -> 62759 bytes final_report/fig_method_diagram.pdf | Bin 0 -> 15100 bytes final_report/fig_method_diagram.png | Bin 0 -> 81057 bytes final_report/fig_module_delta_heatmap.pdf | Bin 0 -> 21009 bytes final_report/fig_module_delta_heatmap.png | Bin 0 -> 86207 bytes final_report/fig_module_heatmap.pdf | Bin 0 -> 13914 bytes final_report/fig_module_heatmap.png | Bin 0 -> 56430 bytes final_report/fig_protocol_bridge.pdf | Bin 0 -> 12682 bytes final_report/fig_protocol_bridge.png | Bin 0 -> 54931 bytes final_report/fig_win_counts.pdf | Bin 0 -> 15560 bytes final_report/fig_win_counts.png | Bin 0 -> 41491 bytes final_report/paper.pdf | Bin 0 -> 131889 bytes final_report/paper.tex | 268 +++ final_report/references.bib | 27 + .../source_final_experiment_report.md | 116 ++ .../source_final_integrated_summary.csv | 31 + 21 files changed, 3032 insertions(+) create mode 100644 final_report/.gitignore create mode 100644 final_report/aaai2026.bst create mode 100644 final_report/aaai2026.sty create mode 100644 final_report/build_report.py create mode 100644 final_report/fig_combined_comparison.pdf create mode 100644 final_report/fig_combined_comparison.png create mode 100644 final_report/fig_method_diagram.pdf create mode 100644 final_report/fig_method_diagram.png create mode 100644 final_report/fig_module_delta_heatmap.pdf create mode 100644 final_report/fig_module_delta_heatmap.png create mode 100644 final_report/fig_module_heatmap.pdf create mode 100644 final_report/fig_module_heatmap.png create mode 100644 final_report/fig_protocol_bridge.pdf create mode 100644 final_report/fig_protocol_bridge.png create mode 100644 final_report/fig_win_counts.pdf create mode 100644 final_report/fig_win_counts.png create mode 100644 final_report/paper.pdf create mode 100644 final_report/paper.tex create mode 100644 final_report/references.bib create mode 100644 final_report/source_final_experiment_report.md create mode 100644 final_report/source_final_integrated_summary.csv diff --git a/final_report/.gitignore b/final_report/.gitignore new file mode 100644 index 000000000..65e51b063 --- /dev/null +++ b/final_report/.gitignore @@ -0,0 +1,2 @@ +aaai_authorkit/ +preview_page_*.png diff --git a/final_report/aaai2026.bst b/final_report/aaai2026.bst new file mode 100644 index 000000000..bc73330ee --- /dev/null +++ b/final_report/aaai2026.bst @@ -0,0 +1,1493 @@ +%% +%% This is file `aaai2026.bst', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% merlin.mbs (with options: `head,ay,nat,ed-au,nm-rev,ed-rev,jnrlst,aunm-semi,mcite,mct-1,mct-x3,keyxyr,dt-beg,yr-per,yrp-per,note-yr,atit-u,volp-sp,num-xser,bkpg-x,add-pub,isbn,ppx,ed,xedn,and-com,and-com-ed,etal-xc,nfss,,{}') +%% merlin.mbs (with options: `tail,ay,nat,ed-au,nm-rev,ed-rev,jnrlst,aunm-semi,mcite,mct-1,mct-x3,keyxyr,dt-beg,yr-per,yrp-per,note-yr,atit-u,volp-sp,num-xser,bkpg-x,add-pub,isbn,ppx,ed,xedn,and-com,and-com-ed,etal-xc,nfss,,{}') +%% ---------------------------------------- +%% *** Natbib-compatible implementation of 'aaai' bib style *** +%% + % =============================================================== + % IMPORTANT NOTICE: + % This bibliographic style (bst) file has been generated from one or + % more master bibliographic style (mbs) files, listed above. + % + % This generated file can be redistributed and/or modified under the terms + % of the LaTeX Project Public License Distributed from CTAN + % archives in directory macros/latex/base/lppl.txt; either + % version 1 of the License, or any later version. + % =============================================================== + % Name and version information of the main mbs file: + % \ProvidesFile{merlin.mbs}[2011/11/18 4.33 (PWD, AO, DPC)] + % For use with BibTeX version 0.99a or later + %------------------------------------------------------------------- + % This bibliography style file is intended for texts in ENGLISH + % This is an author-year citation style bibliography. As such, it is + % non-standard LaTeX, and requires a special package file to function properly. + % Such a package is natbib.sty by Patrick W. Daly + % The form of the \bibitem entries is + % \bibitem[Jones et al.(1990)]{key}... + % \bibitem[Jones et al.(1990)Jones, Baker, and Smith]{key}... + % The essential feature is that the label (the part in brackets) consists + % of the author names, as they should appear in the citation, with the year + % in parentheses following. There must be no space before the opening + % parenthesis! + % With natbib v5.3, a full list of authors may also follow the year. + % In natbib.sty, it is possible to define the type of enclosures that is + % really wanted (brackets or parentheses), but in either case, there must + % be parentheses in the label. + % The \cite command functions as follows: + % \citet{key} ==>> Jones et al. (1990) + % \citet*{key} ==>> Jones, Baker, and Smith (1990) + % \citep{key} ==>> (Jones et al., 1990) + % \citep*{key} ==>> (Jones, Baker, and Smith, 1990) + % \citep[chap. 2]{key} ==>> (Jones et al., 1990, chap. 2) + % \citep[e.g.][]{key} ==>> (e.g. Jones et al., 1990) + % \citep[e.g.][p. 32]{key} ==>> (e.g. Jones et al., 1990, p. 32) + % \citeauthor{key} ==>> Jones et al. + % \citeauthor*{key} ==>> Jones, Baker, and Smith + % \citeyear{key} ==>> 1990 + %--------------------------------------------------------------------- + +ENTRY + { address + archivePrefix + author + booktitle + chapter + edition + editor + eid + eprint + howpublished + institution + isbn + journal + key + month + note + number + organization + pages + publisher + school + series + title + type + volume + year + } + {} + { label extra.label sort.label short.list } +INTEGERS { output.state before.all mid.sentence after.sentence after.block } +FUNCTION {init.state.consts} +{ #0 'before.all := + #1 'mid.sentence := + #2 'after.sentence := + #3 'after.block := +} +STRINGS { s t} +FUNCTION {output.nonnull} +{ 's := + output.state mid.sentence = + { ", " * write$ } + { output.state after.block = + { add.period$ write$ + newline$ + "\newblock " write$ + } + { output.state before.all = + 'write$ + { add.period$ " " * write$ } + if$ + } + if$ + mid.sentence 'output.state := + } + if$ + s +} +FUNCTION {output} +{ duplicate$ empty$ + 'pop$ + 'output.nonnull + if$ +} +FUNCTION {output.check} +{ 't := + duplicate$ empty$ + { pop$ "empty " t * " in " * cite$ * warning$ } + 'output.nonnull + if$ +} +FUNCTION {fin.entry} +{ add.period$ + write$ + newline$ +} + +FUNCTION {new.block} +{ output.state before.all = + 'skip$ + { after.block 'output.state := } + if$ +} +FUNCTION {new.sentence} +{ output.state after.block = + 'skip$ + { output.state before.all = + 'skip$ + { after.sentence 'output.state := } + if$ + } + if$ +} +FUNCTION {add.blank} +{ " " * before.all 'output.state := +} + +FUNCTION {date.block} +{ + new.block +} + +FUNCTION {not} +{ { #0 } + { #1 } + if$ +} +FUNCTION {and} +{ 'skip$ + { pop$ #0 } + if$ +} +FUNCTION {or} +{ { pop$ #1 } + 'skip$ + if$ +} +FUNCTION {new.block.checkb} +{ empty$ + swap$ empty$ + and + 'skip$ + 'new.block + if$ +} +FUNCTION {field.or.null} +{ duplicate$ empty$ + { pop$ "" } + 'skip$ + if$ +} +FUNCTION {emphasize} +{ duplicate$ empty$ + { pop$ "" } + { "\emph{" swap$ * "}" * } + if$ +} +FUNCTION {tie.or.space.prefix} +{ duplicate$ text.length$ #3 < + { "~" } + { " " } + if$ + swap$ +} + +FUNCTION {capitalize} +{ "u" change.case$ "t" change.case$ } + +FUNCTION {space.word} +{ " " swap$ * " " * } + % Here are the language-specific definitions for explicit words. + % Each function has a name bbl.xxx where xxx is the English word. + % The language selected here is ENGLISH +FUNCTION {bbl.and} +{ "and"} + +FUNCTION {bbl.etal} +{ "et~al." } + +FUNCTION {bbl.editors} +{ "eds." } + +FUNCTION {bbl.editor} +{ "ed." } + +FUNCTION {bbl.edby} +{ "edited by" } + +FUNCTION {bbl.edition} +{ "edition" } + +FUNCTION {bbl.volume} +{ "volume" } + +FUNCTION {bbl.of} +{ "of" } + +FUNCTION {bbl.number} +{ "number" } + +FUNCTION {bbl.nr} +{ "no." } + +FUNCTION {bbl.in} +{ "in" } + +FUNCTION {bbl.pages} +{ "" } + +FUNCTION {bbl.page} +{ "" } + +FUNCTION {bbl.chapter} +{ "chapter" } + +FUNCTION {bbl.techrep} +{ "Technical Report" } + +FUNCTION {bbl.mthesis} +{ "Master's thesis" } + +FUNCTION {bbl.phdthesis} +{ "Ph.D. thesis" } + +MACRO {jan} {"January"} + +MACRO {feb} {"February"} + +MACRO {mar} {"March"} + +MACRO {apr} {"April"} + +MACRO {may} {"May"} + +MACRO {jun} {"June"} + +MACRO {jul} {"July"} + +MACRO {aug} {"August"} + +MACRO {sep} {"September"} + +MACRO {oct} {"October"} + +MACRO {nov} {"November"} + +MACRO {dec} {"December"} + +MACRO {acmcs} {"ACM Computing Surveys"} + +MACRO {acta} {"Acta Informatica"} + +MACRO {cacm} {"Communications of the ACM"} + +MACRO {ibmjrd} {"IBM Journal of Research and Development"} + +MACRO {ibmsj} {"IBM Systems Journal"} + +MACRO {ieeese} {"IEEE Transactions on Software Engineering"} + +MACRO {ieeetc} {"IEEE Transactions on Computers"} + +MACRO {ieeetcad} + {"IEEE Transactions on Computer-Aided Design of Integrated Circuits"} + +MACRO {ipl} {"Information Processing Letters"} + +MACRO {jacm} {"Journal of the ACM"} + +MACRO {jcss} {"Journal of Computer and System Sciences"} + +MACRO {scp} {"Science of Computer Programming"} + +MACRO {sicomp} {"SIAM Journal on Computing"} + +MACRO {tocs} {"ACM Transactions on Computer Systems"} + +MACRO {tods} {"ACM Transactions on Database Systems"} + +MACRO {tog} {"ACM Transactions on Graphics"} + +MACRO {toms} {"ACM Transactions on Mathematical Software"} + +MACRO {toois} {"ACM Transactions on Office Information Systems"} + +MACRO {toplas} {"ACM Transactions on Programming Languages and Systems"} + +MACRO {tcs} {"Theoretical Computer Science"} +FUNCTION {bibinfo.check} +{ swap$ + duplicate$ missing$ + { + pop$ pop$ + "" + } + { duplicate$ empty$ + { + swap$ pop$ + } + { swap$ + pop$ + } + if$ + } + if$ +} +FUNCTION {bibinfo.warn} +{ swap$ + duplicate$ missing$ + { + swap$ "missing " swap$ * " in " * cite$ * warning$ pop$ + "" + } + { duplicate$ empty$ + { + swap$ "empty " swap$ * " in " * cite$ * warning$ + } + { swap$ + pop$ + } + if$ + } + if$ +} +FUNCTION {format.eprint} +{ eprint duplicate$ empty$ + 'skip$ + { archivePrefix duplicate$ empty$ + 'skip$ + { ":" * swap$ } + if$ + * "." * + } + if$ +} +INTEGERS { nameptr namesleft numnames } + + +STRINGS { bibinfo} + +FUNCTION {format.names} +{ 'bibinfo := + duplicate$ empty$ 'skip$ { + 's := + "" 't := + #1 'nameptr := + s num.names$ 'numnames := + numnames 'namesleft := + { namesleft #0 > } + { s nameptr + "{vv~}{ll}{, f.}{, jj}" + format.name$ + bibinfo bibinfo.check + 't := + nameptr #1 > + { + namesleft #1 > + { "; " * t * } + { + s nameptr "{ll}" format.name$ duplicate$ "others" = + { 't := } + { pop$ } + if$ + ";" * + t "others" = + { + " " * bbl.etal * + } + { + bbl.and + space.word * t * + } + if$ + } + if$ + } + 't + if$ + nameptr #1 + 'nameptr := + namesleft #1 - 'namesleft := + } + while$ + } if$ +} +FUNCTION {format.names.ed} +{ + format.names +} +FUNCTION {format.key} +{ empty$ + { key field.or.null } + { "" } + if$ +} + +FUNCTION {format.authors} +{ author "author" format.names +} +FUNCTION {get.bbl.editor} +{ editor num.names$ #1 > 'bbl.editors 'bbl.editor if$ } + +FUNCTION {format.editors} +{ editor "editor" format.names duplicate$ empty$ 'skip$ + { + "," * + " " * + get.bbl.editor + * + } + if$ +} +FUNCTION {format.isbn} +{ isbn "isbn" bibinfo.check + duplicate$ empty$ 'skip$ + { + new.block + "ISBN " swap$ * + } + if$ +} + +FUNCTION {format.note} +{ + note empty$ + { "" } + { note #1 #1 substring$ + duplicate$ "{" = + 'skip$ + { output.state mid.sentence = + { "l" } + { "u" } + if$ + change.case$ + } + if$ + note #2 global.max$ substring$ * "note" bibinfo.check + } + if$ +} + +FUNCTION {format.title} +{ title + "title" bibinfo.check +} +FUNCTION {format.full.names} +{'s := + "" 't := + #1 'nameptr := + s num.names$ 'numnames := + numnames 'namesleft := + { namesleft #0 > } + { s nameptr + "{vv~}{ll}" format.name$ + 't := + nameptr #1 > + { + namesleft #1 > + { ", " * t * } + { + s nameptr "{ll}" format.name$ duplicate$ "others" = + { 't := } + { pop$ } + if$ + t "others" = + { + " " * bbl.etal * + } + { + numnames #2 > + { "," * } + 'skip$ + if$ + bbl.and + space.word * t * + } + if$ + } + if$ + } + 't + if$ + nameptr #1 + 'nameptr := + namesleft #1 - 'namesleft := + } + while$ +} + +FUNCTION {author.editor.key.full} +{ author empty$ + { editor empty$ + { key empty$ + { cite$ #1 #3 substring$ } + 'key + if$ + } + { editor format.full.names } + if$ + } + { author format.full.names } + if$ +} + +FUNCTION {author.key.full} +{ author empty$ + { key empty$ + { cite$ #1 #3 substring$ } + 'key + if$ + } + { author format.full.names } + if$ +} + +FUNCTION {editor.key.full} +{ editor empty$ + { key empty$ + { cite$ #1 #3 substring$ } + 'key + if$ + } + { editor format.full.names } + if$ +} + +FUNCTION {make.full.names} +{ type$ "book" = + type$ "inbook" = + or + 'author.editor.key.full + { type$ "proceedings" = + 'editor.key.full + 'author.key.full + if$ + } + if$ +} + +FUNCTION {output.bibitem} +{ newline$ + "\bibitem[{" write$ + label write$ + ")" make.full.names duplicate$ short.list = + { pop$ } + { * } + if$ + "}]{" * write$ + cite$ write$ + "}" write$ + newline$ + "" + before.all 'output.state := +} + +FUNCTION {n.dashify} +{ + 't := + "" + { t empty$ not } + { t #1 #1 substring$ "-" = + { t #1 #2 substring$ "--" = not + { "--" * + t #2 global.max$ substring$ 't := + } + { { t #1 #1 substring$ "-" = } + { "-" * + t #2 global.max$ substring$ 't := + } + while$ + } + if$ + } + { t #1 #1 substring$ * + t #2 global.max$ substring$ 't := + } + if$ + } + while$ +} + +FUNCTION {word.in} +{ bbl.in capitalize + " " * } + +FUNCTION {format.date} +{ year "year" bibinfo.check duplicate$ empty$ + { + "empty year in " cite$ * "; set to ????" * warning$ + pop$ "????" + } + 'skip$ + if$ + extra.label * + before.all 'output.state := + after.sentence 'output.state := +} +FUNCTION {format.btitle} +{ title "title" bibinfo.check + duplicate$ empty$ 'skip$ + { + emphasize + } + if$ +} +FUNCTION {either.or.check} +{ empty$ + 'pop$ + { "can't use both " swap$ * " fields in " * cite$ * warning$ } + if$ +} +FUNCTION {format.bvolume} +{ volume empty$ + { "" } + { bbl.volume volume tie.or.space.prefix + "volume" bibinfo.check * * + series "series" bibinfo.check + duplicate$ empty$ 'pop$ + { swap$ bbl.of space.word * swap$ + emphasize * } + if$ + "volume and number" number either.or.check + } + if$ +} +FUNCTION {format.number.series} +{ volume empty$ + { number empty$ + { series field.or.null } + { series empty$ + { number "number" bibinfo.check } + { output.state mid.sentence = + { bbl.number } + { bbl.number capitalize } + if$ + number tie.or.space.prefix "number" bibinfo.check * * + bbl.in space.word * + series "series" bibinfo.check * + } + if$ + } + if$ + } + { "" } + if$ +} + +FUNCTION {format.edition} +{ edition duplicate$ empty$ 'skip$ + { + output.state mid.sentence = + { "l" } + { "t" } + if$ change.case$ + "edition" bibinfo.check + " " * bbl.edition * + } + if$ +} +INTEGERS { multiresult } +FUNCTION {multi.page.check} +{ 't := + #0 'multiresult := + { multiresult not + t empty$ not + and + } + { t #1 #1 substring$ + duplicate$ "-" = + swap$ duplicate$ "," = + swap$ "+" = + or or + { #1 'multiresult := } + { t #2 global.max$ substring$ 't := } + if$ + } + while$ + multiresult +} +FUNCTION {format.pages} +{ pages duplicate$ empty$ 'skip$ + { duplicate$ multi.page.check + { + n.dashify + } + { + } + if$ + "pages" bibinfo.check + } + if$ +} +FUNCTION {format.journal.pages} +{ pages duplicate$ empty$ 'pop$ + { swap$ duplicate$ empty$ + { pop$ pop$ format.pages } + { + ": " * + swap$ + n.dashify + "pages" bibinfo.check + * + } + if$ + } + if$ +} +FUNCTION {format.journal.eid} +{ eid "eid" bibinfo.check + duplicate$ empty$ 'pop$ + { swap$ duplicate$ empty$ 'skip$ + { + ": " * + } + if$ + swap$ * + } + if$ +} +FUNCTION {format.vol.num.pages} +{ volume field.or.null + duplicate$ empty$ 'skip$ + { + "volume" bibinfo.check + } + if$ + number "number" bibinfo.check duplicate$ empty$ 'skip$ + { + swap$ duplicate$ empty$ + { "there's a number but no volume in " cite$ * warning$ } + 'skip$ + if$ + swap$ + "(" swap$ * ")" * + } + if$ * + eid empty$ + { format.journal.pages } + { format.journal.eid } + if$ +} + +FUNCTION {format.chapter.pages} +{ chapter empty$ + 'format.pages + { type empty$ + { bbl.chapter } + { type "l" change.case$ + "type" bibinfo.check + } + if$ + chapter tie.or.space.prefix + "chapter" bibinfo.check + * * + pages empty$ + 'skip$ + { ", " * format.pages * } + if$ + } + if$ +} + +FUNCTION {format.booktitle} +{ + booktitle "booktitle" bibinfo.check + emphasize +} +FUNCTION {format.in.ed.booktitle} +{ format.booktitle duplicate$ empty$ 'skip$ + { + editor "editor" format.names.ed duplicate$ empty$ 'pop$ + { + "," * + " " * + get.bbl.editor + ", " * + * swap$ + * } + if$ + word.in swap$ * + } + if$ +} +FUNCTION {format.thesis.type} +{ type duplicate$ empty$ + 'pop$ + { swap$ pop$ + "t" change.case$ "type" bibinfo.check + } + if$ +} +FUNCTION {format.tr.number} +{ number "number" bibinfo.check + type duplicate$ empty$ + { pop$ bbl.techrep } + 'skip$ + if$ + "type" bibinfo.check + swap$ duplicate$ empty$ + { pop$ "t" change.case$ } + { tie.or.space.prefix * * } + if$ +} +FUNCTION {format.article.crossref} +{ + word.in + " \cite{" * crossref * "}" * +} +FUNCTION {format.book.crossref} +{ volume duplicate$ empty$ + { "empty volume in " cite$ * "'s crossref of " * crossref * warning$ + pop$ word.in + } + { bbl.volume + capitalize + swap$ tie.or.space.prefix "volume" bibinfo.check * * bbl.of space.word * + } + if$ + " \cite{" * crossref * "}" * +} +FUNCTION {format.incoll.inproc.crossref} +{ + word.in + " \cite{" * crossref * "}" * +} +FUNCTION {format.org.or.pub} +{ 't := + "" + address empty$ t empty$ and + 'skip$ + { + address "address" bibinfo.check * + t empty$ + 'skip$ + { address empty$ + 'skip$ + { ": " * } + if$ + t * + } + if$ + } + if$ +} +FUNCTION {format.publisher.address} +{ publisher "publisher" bibinfo.warn format.org.or.pub +} + +FUNCTION {format.organization.address} +{ organization "organization" bibinfo.check format.org.or.pub +} + +FUNCTION {article} +{ output.bibitem + format.authors "author" output.check + author format.key output + format.date "year" output.check + date.block + format.title "title" output.check + new.block + crossref missing$ + { + journal + "journal" bibinfo.check + emphasize + "journal" output.check + format.vol.num.pages output + } + { format.article.crossref output.nonnull + format.pages output + } + if$ + new.block + format.note output + fin.entry +} +FUNCTION {book} +{ output.bibitem + author empty$ + { format.editors "author and editor" output.check + editor format.key output + } + { format.authors output.nonnull + crossref missing$ + { "author and editor" editor either.or.check } + 'skip$ + if$ + } + if$ + format.date "year" output.check + date.block + format.btitle "title" output.check + crossref missing$ + { format.bvolume output + new.block + format.number.series output + new.sentence + format.publisher.address output + } + { + new.block + format.book.crossref output.nonnull + } + if$ + format.edition output + format.isbn output + new.block + format.note output + fin.entry +} +FUNCTION {booklet} +{ output.bibitem + format.authors output + author format.key output + format.date "year" output.check + date.block + format.title "title" output.check + new.block + howpublished "howpublished" bibinfo.check output + address "address" bibinfo.check output + format.isbn output + new.block + format.note output + fin.entry +} + +FUNCTION {inbook} +{ output.bibitem + author empty$ + { format.editors "author and editor" output.check + editor format.key output + } + { format.authors output.nonnull + crossref missing$ + { "author and editor" editor either.or.check } + 'skip$ + if$ + } + if$ + format.date "year" output.check + date.block + format.btitle "title" output.check + crossref missing$ + { + format.bvolume output + format.chapter.pages "chapter and pages" output.check + new.block + format.number.series output + new.sentence + format.publisher.address output + } + { + format.chapter.pages "chapter and pages" output.check + new.block + format.book.crossref output.nonnull + } + if$ + format.edition output + crossref missing$ + { format.isbn output } + 'skip$ + if$ + new.block + format.note output + fin.entry +} + +FUNCTION {incollection} +{ output.bibitem + format.authors "author" output.check + author format.key output + format.date "year" output.check + date.block + format.title "title" output.check + new.block + crossref missing$ + { format.in.ed.booktitle "booktitle" output.check + format.bvolume output + format.number.series output + format.chapter.pages output + new.sentence + format.publisher.address output + format.edition output + format.isbn output + } + { format.incoll.inproc.crossref output.nonnull + format.chapter.pages output + } + if$ + new.block + format.note output + fin.entry +} +FUNCTION {inproceedings} +{ output.bibitem + format.authors "author" output.check + author format.key output + format.date "year" output.check + date.block + format.title "title" output.check + new.block + crossref missing$ + { format.in.ed.booktitle "booktitle" output.check + format.bvolume output + format.number.series output + format.pages output + new.sentence + publisher empty$ + { format.organization.address output } + { organization "organization" bibinfo.check output + format.publisher.address output + } + if$ + format.isbn output + } + { format.incoll.inproc.crossref output.nonnull + format.pages output + } + if$ + new.block + format.note output + fin.entry +} +FUNCTION {conference} { inproceedings } +FUNCTION {manual} +{ output.bibitem + format.authors output + author format.key output + format.date "year" output.check + date.block + format.btitle "title" output.check + organization address new.block.checkb + organization "organization" bibinfo.check output + address "address" bibinfo.check output + format.edition output + new.block + format.note output + fin.entry +} + +FUNCTION {mastersthesis} +{ output.bibitem + format.authors "author" output.check + author format.key output + format.date "year" output.check + date.block + format.btitle + "title" output.check + new.block + bbl.mthesis format.thesis.type output.nonnull + school "school" bibinfo.warn output + address "address" bibinfo.check output + new.block + format.note output + fin.entry +} + +FUNCTION {misc} +{ output.bibitem + format.authors output + author format.key output + format.date "year" output.check + date.block + format.title output + new.block + howpublished "howpublished" bibinfo.check output + new.block + format.note output + format.eprint output + fin.entry +} +FUNCTION {phdthesis} +{ output.bibitem + format.authors "author" output.check + author format.key output + format.date "year" output.check + date.block + format.btitle + "title" output.check + new.block + bbl.phdthesis format.thesis.type output.nonnull + school "school" bibinfo.warn output + address "address" bibinfo.check output + new.block + format.note output + fin.entry +} + +FUNCTION {proceedings} +{ output.bibitem + format.editors output + editor format.key output + format.date "year" output.check + date.block + format.btitle "title" output.check + format.bvolume output + format.number.series output + new.sentence + publisher empty$ + { format.organization.address output } + { organization "organization" bibinfo.check output + format.publisher.address output + } + if$ + format.isbn output + new.block + format.note output + fin.entry +} + +FUNCTION {techreport} +{ output.bibitem + format.authors "author" output.check + author format.key output + format.date "year" output.check + date.block + format.title + "title" output.check + new.block + format.tr.number output.nonnull + institution "institution" bibinfo.warn output + address "address" bibinfo.check output + new.block + format.note output + fin.entry +} + +FUNCTION {unpublished} +{ output.bibitem + format.authors "author" output.check + author format.key output + format.date "year" output.check + date.block + format.title "title" output.check + new.block + format.note "note" output.check + fin.entry +} + +FUNCTION {default.type} { misc } +READ +FUNCTION {sortify} +{ purify$ + "l" change.case$ +} +INTEGERS { len } +FUNCTION {chop.word} +{ 's := + 'len := + s #1 len substring$ = + { s len #1 + global.max$ substring$ } + 's + if$ +} +FUNCTION {format.lab.names} +{'s := + "" 't := + #1 'nameptr := + s num.names$ 'numnames := + numnames 'namesleft := + { namesleft #0 > } + { s nameptr + "{vv~}{ll}" format.name$ + 't := + nameptr #1 > + { + nameptr #2 = + numnames #3 > and + { "others" 't := + #1 'namesleft := } + 'skip$ + if$ + namesleft #1 > + { ", " * t * } + { + s nameptr "{ll}" format.name$ duplicate$ "others" = + { 't := } + { pop$ } + if$ + t "others" = + { + " " * bbl.etal * + } + { + numnames #2 > + { "," * } + 'skip$ + if$ + bbl.and + space.word * t * + } + if$ + } + if$ + } + 't + if$ + nameptr #1 + 'nameptr := + namesleft #1 - 'namesleft := + } + while$ +} + +FUNCTION {author.key.label} +{ author empty$ + { key empty$ + { cite$ #1 #3 substring$ } + 'key + if$ + } + { author format.lab.names } + if$ +} + +FUNCTION {author.editor.key.label} +{ author empty$ + { editor empty$ + { key empty$ + { cite$ #1 #3 substring$ } + 'key + if$ + } + { editor format.lab.names } + if$ + } + { author format.lab.names } + if$ +} + +FUNCTION {editor.key.label} +{ editor empty$ + { key empty$ + { cite$ #1 #3 substring$ } + 'key + if$ + } + { editor format.lab.names } + if$ +} + +FUNCTION {calc.short.authors} +{ type$ "book" = + type$ "inbook" = + or + 'author.editor.key.label + { type$ "proceedings" = + 'editor.key.label + 'author.key.label + if$ + } + if$ + 'short.list := +} + +FUNCTION {calc.label} +{ calc.short.authors + short.list + "(" + * + year duplicate$ empty$ + short.list key field.or.null = or + { pop$ "" } + 'skip$ + if$ + * + 'label := +} + +FUNCTION {sort.format.names} +{ 's := + #1 'nameptr := + "" + s num.names$ 'numnames := + numnames 'namesleft := + { namesleft #0 > } + { s nameptr + "{vv{ } }{ll{ }}{ f{ }}{ jj{ }}" + format.name$ 't := + nameptr #1 > + { + " " * + namesleft #1 = t "others" = and + { "zzzzz" 't := } + 'skip$ + if$ + t sortify * + } + { t sortify * } + if$ + nameptr #1 + 'nameptr := + namesleft #1 - 'namesleft := + } + while$ +} + +FUNCTION {sort.format.title} +{ 't := + "A " #2 + "An " #3 + "The " #4 t chop.word + chop.word + chop.word + sortify + #1 global.max$ substring$ +} +FUNCTION {author.sort} +{ author empty$ + { key empty$ + { "to sort, need author or key in " cite$ * warning$ + "" + } + { key sortify } + if$ + } + { author sort.format.names } + if$ +} +FUNCTION {author.editor.sort} +{ author empty$ + { editor empty$ + { key empty$ + { "to sort, need author, editor, or key in " cite$ * warning$ + "" + } + { key sortify } + if$ + } + { editor sort.format.names } + if$ + } + { author sort.format.names } + if$ +} +FUNCTION {editor.sort} +{ editor empty$ + { key empty$ + { "to sort, need editor or key in " cite$ * warning$ + "" + } + { key sortify } + if$ + } + { editor sort.format.names } + if$ +} +FUNCTION {presort} +{ calc.label + label sortify + " " + * + type$ "book" = + type$ "inbook" = + or + 'author.editor.sort + { type$ "proceedings" = + 'editor.sort + 'author.sort + if$ + } + if$ + #1 entry.max$ substring$ + 'sort.label := + sort.label + * + " " + * + title field.or.null + sort.format.title + * + #1 entry.max$ substring$ + 'sort.key$ := +} + +ITERATE {presort} +SORT +STRINGS { last.label next.extra } +INTEGERS { last.extra.num last.extra.num.extended last.extra.num.blank number.label } +FUNCTION {initialize.extra.label.stuff} +{ #0 int.to.chr$ 'last.label := + "" 'next.extra := + #0 'last.extra.num := + "a" chr.to.int$ #1 - 'last.extra.num.blank := + last.extra.num.blank 'last.extra.num.extended := + #0 'number.label := +} +FUNCTION {forward.pass} +{ last.label label = + { last.extra.num #1 + 'last.extra.num := + last.extra.num "z" chr.to.int$ > + { "a" chr.to.int$ 'last.extra.num := + last.extra.num.extended #1 + 'last.extra.num.extended := + } + 'skip$ + if$ + last.extra.num.extended last.extra.num.blank > + { last.extra.num.extended int.to.chr$ + last.extra.num int.to.chr$ + * 'extra.label := } + { last.extra.num int.to.chr$ 'extra.label := } + if$ + } + { "a" chr.to.int$ 'last.extra.num := + "" 'extra.label := + label 'last.label := + } + if$ + number.label #1 + 'number.label := +} +FUNCTION {reverse.pass} +{ next.extra "b" = + { "a" 'extra.label := } + 'skip$ + if$ + extra.label 'next.extra := + extra.label + duplicate$ empty$ + 'skip$ + { "{\natexlab{" swap$ * "}}" * } + if$ + 'extra.label := + label extra.label * 'label := +} +EXECUTE {initialize.extra.label.stuff} +ITERATE {forward.pass} +REVERSE {reverse.pass} +FUNCTION {bib.sort.order} +{ sort.label + " " + * + year field.or.null sortify + * + " " + * + title field.or.null + sort.format.title + * + #1 entry.max$ substring$ + 'sort.key$ := +} +ITERATE {bib.sort.order} +SORT +FUNCTION {begin.bib} +{ preamble$ empty$ + 'skip$ + { preamble$ write$ newline$ } + if$ + "\begin{thebibliography}{" number.label int.to.str$ * "}" * + write$ newline$ + "\providecommand{\natexlab}[1]{#1}" + write$ newline$ +} +EXECUTE {begin.bib} +EXECUTE {init.state.consts} +ITERATE {call.type$} +FUNCTION {end.bib} +{ newline$ + "\end{thebibliography}" write$ newline$ +} +EXECUTE {end.bib} +%% End of customized bst file +%% +%% End of file `aaai2026.bst'. diff --git a/final_report/aaai2026.sty b/final_report/aaai2026.sty new file mode 100644 index 000000000..989b76119 --- /dev/null +++ b/final_report/aaai2026.sty @@ -0,0 +1,318 @@ +\NeedsTeXFormat{LaTeX2e}% +\ProvidesPackage{aaai2026}[2026/06/17 AAAI 2026 Submission format]% +\def\year{2026}% +\typeout{Conference Style for AAAI for LaTeX 2e -- version for submission}% +% +\def\copyright@on{T} +\def\showauthors@on{T} +\def\nocopyright{\gdef\copyright@on{}} % Copyright notice is required for camera-ready only. +\DeclareOption{submission}{% + \gdef\copyright@on{}% + \gdef\showauthors@on{}% + \long\gdef\pdfinfo #1{\relax}% +}% +\DeclareOption{draft}{% + \gdef\copyright@on{}% +}% +\ProcessOptions\relax% +% WARNING: IF YOU ARE USING THIS STYLE SHEET FOR AN AAAI PUBLICATION, YOU +% MAY NOT MODIFY IT FOR ANY REASON. MODIFICATIONS (IN YOUR SOURCE +% OR IN THIS STYLE SHEET WILL RESULT IN REJECTION OF YOUR PAPER). +% +% WARNING: This style is NOT guaranteed to work. It is provided in the +% hope that it might make the preparation of papers easier, but this style +% file is provided "as is" without warranty of any kind, either express or +% implied, including but not limited to the implied warranties of +% merchantability, fitness for a particular purpose, or noninfringement. +% You use this style file at your own risk. Standard disclaimers apply. +% There are undoubtably bugs in this style. If you would like to submit +% bug fixes, improvements, etc. please let us know. Please use the contact form +% at www.aaai.org. +% +% Do not use this file unless you are an experienced LaTeX user. +% +% PHYSICAL PAGE LAYOUT +\setlength\topmargin{-0.25in} \setlength\oddsidemargin{-0.25in} +\setlength\textheight{9.0in} \setlength\textwidth{7.0in} +\setlength\columnsep{0.375in} \newlength\titlebox \setlength\titlebox{2.25in} +\setlength\headheight{0pt} \setlength\headsep{0pt} +%\setlength\footheight{0pt} \setlength\footskip{0pt} +\thispagestyle{empty} \pagestyle{empty} +\flushbottom \twocolumn \sloppy +% We're never going to need a table of contents, so just flush it to +% save space --- suggested by drstrip@sandia-2 +\def\addcontentsline#1#2#3{} +% gf: PRINT COPYRIGHT NOTICE +\def\copyright@year{\number\year} +\def\copyright@text{Copyright \copyright\space \copyright@year, +Association for the Advancement of Artificial Intelligence (www.aaai.org). +All rights reserved.} +\def\copyrighttext#1{\gdef\copyright@on{T}\gdef\copyright@text{#1}} +\def\copyrightyear#1{\gdef\copyright@on{T}\gdef\copyright@year{#1}} +% gf: End changes for copyright notice (used in \maketitle, below) +% Title stuff, taken from deproc. +% +\def\maketitle{% + \par% + \begingroup % to make the footnote style local to the title + \def\thefootnote{\fnsymbol{footnote}} + \twocolumn[\@maketitle] \@thanks% + \endgroup% + % Insert copyright slug unless turned off + \if T\copyright@on\insert\footins{\noindent\footnotesize\copyright@text}\fi% + % + \setcounter{footnote}{0}% + \let\maketitle\relax% + \let\@maketitle\relax% + \gdef\@thanks{}% + \gdef\@author{}% + \gdef\@title{}% + \let\thanks\relax% +}% +\long\gdef\affiliations #1{ \def \affiliations_{\if T\showauthors@on#1\fi}}% +% +\def\@maketitle{% + \def\theauthors{\if T\showauthors@on\@author\else Anonymous submission\fi} + \newcounter{eqfn}\setcounter{eqfn}{0}% + \newsavebox{\titlearea} + \sbox{\titlearea}{ + \let\footnote\relax\let\thanks\relax% + \setcounter{footnote}{0}% + \def\equalcontrib{% + \ifnum\value{eqfn}=0% + \footnote{These authors contributed equally.}% + \setcounter{eqfn}{\value{footnote}}% + \else% + \footnotemark[\value{eqfn}]% + \fi% + }% + \vbox{% + \hsize\textwidth% + \linewidth\hsize% + \vskip 0.625in minus 0.125in% + \centering% + {\LARGE\bf \@title \par}% + \vskip 0.1in plus 0.5fil minus 0.05in% + {\Large{\textbf{\theauthors\ifhmode\\\fi}}}% + \vskip .2em plus 0.25fil% + {\normalsize \affiliations_\ifhmode\\\fi}% + \vskip 1em plus 2fil% + }% + }% +% + \newlength\actualheight% + \settoheight{\actualheight}{\usebox{\titlearea}}% + \ifdim\actualheight>\titlebox% + \setlength{\titlebox}{\actualheight}% + \fi% +% + \vbox to \titlebox {% + \let\footnote\thanks\relax% + \setcounter{footnote}{0}% + \def\equalcontrib{% + \ifnum\value{eqfn}=0% + \footnote{These authors contributed equally.}% + \setcounter{eqfn}{\value{footnote}}% + \else% + \footnotemark[\value{eqfn}]% + \fi% + }% + \hsize\textwidth% + \linewidth\hsize% + \vskip 0.625in minus 0.125in% + \centering% + {\LARGE\bf \@title \par}% + \vskip 0.1in plus 0.5fil minus 0.05in% + {\Large{\textbf{\theauthors\ifhmode\\\fi}}}% + \vskip .2em plus 0.25fil% + {\normalsize \affiliations_\ifhmode\\\fi}% + \vskip 1em plus 2fil% + }% +}% +% +\renewenvironment{abstract}{% + \centerline{\bf Abstract}% + \vspace{0.5ex}% + \setlength{\leftmargini}{10pt}% + \begin{quote}% + \small% +}{% + \par% + \end{quote}% + \vskip 1ex% +}% +\newenvironment{links}{% + \newcommand{\link}[2]{\par\textbf{##1} --- \url{##2}}% + \setlength{\hangindent}{10pt}% + \setlength{\parskip}{2pt}% + \begin{flushleft}% +}{% + \end{flushleft}% + \vskip 1ex% +}% +% jsp added: +\def\pubnote#1{ + \thispagestyle{myheadings}% + \pagestyle{myheadings}% + \markboth{#1}{#1}% + \setlength\headheight{10pt}% + \setlength\headsep{10pt}% +}% +% +% SECTIONS with less space +\def\section{\@startsection {section}{1}{\z@}{-2.0ex plus +-0.5ex minus -.2ex}{3pt plus 2pt minus 1pt}{\Large\bf\centering}} +\def\subsection{\@startsection{subsection}{2}{\z@}{-2.0ex plus +-0.5ex minus -.2ex}{3pt plus 2pt minus 1pt}{\large\bf\raggedright}} +\def\subsubsection{\@startsection{subparagraph}{3}{\z@}{-6pt plus +%%% DIEGO changed: 29/11/2009 +%% 2pt minus 1pt}{-1em}{\normalsize\bf}} +-2pt minus -1pt}{-1em}{\normalsize\bf}} +%%% END changed +\renewcommand\paragraph{\@startsection{paragraph}{4}{\z@}{-6pt plus -2pt minus -1pt}{-1em}{\normalsize\bf}}% +\setcounter{secnumdepth}{0} +% add period to section (but not subsection) numbers, reduce space after +%\renewcommand{\thesection} +% {\arabic{section}.\hskip-0.6em} +%\renewcommand{\thesubsection} +% {\arabic{section}.\arabic{subsection}\hskip-0.6em} +% FOOTNOTES +\footnotesep 6.65pt % +\skip\footins 9pt plus 4pt minus 2pt +\def\footnoterule{\kern-3pt \hrule width 5pc \kern 2.6pt } +\setcounter{footnote}{0} +% LISTS AND PARAGRAPHS +\parindent 10pt +\topsep 4pt plus 1pt minus 2pt +\partopsep 1pt plus 0.5pt minus 0.5pt +\itemsep 0.5pt plus 1pt minus 0.5pt +\parsep 2pt plus 1pt minus 0.5pt +\leftmargin 10pt \leftmargini 13pt \leftmarginii 10pt \leftmarginiii 5pt \leftmarginiv 5pt \leftmarginv 5pt \leftmarginvi 5pt +\labelwidth\leftmargini\advance\labelwidth-\labelsep \labelsep 5pt +\def\@listi{\leftmargin\leftmargini} +\def\@listii{\leftmargin\leftmarginii +\labelwidth\leftmarginii\advance\labelwidth-\labelsep +\topsep 2pt plus 1pt minus 0.5pt +\parsep 1pt plus 0.5pt minus 0.5pt +\itemsep \parsep} +\def\@listiii{\leftmargin\leftmarginiii +\labelwidth\leftmarginiii\advance\labelwidth-\labelsep +\topsep 1pt plus 0.5pt minus 0.5pt +\parsep \z@ +\partopsep 0.5pt plus 0pt minus 0.5pt +\itemsep \topsep} +\def\@listiv{\leftmargin\leftmarginiv +\labelwidth\leftmarginiv\advance\labelwidth-\labelsep} +\def\@listv{\leftmargin\leftmarginv +\labelwidth\leftmarginv\advance\labelwidth-\labelsep} +\def\@listvi{\leftmargin\leftmarginvi +\labelwidth\leftmarginvi\advance\labelwidth-\labelsep} +\abovedisplayskip 7pt plus2pt minus5pt% +\belowdisplayskip \abovedisplayskip +\abovedisplayshortskip 0pt plus3pt% +\belowdisplayshortskip 4pt plus3pt minus3pt% +% Less leading in most fonts (due to the narrow columns) +% The choices were between 1-pt and 1.5-pt leading +\def\normalsize{\@setfontsize\normalsize\@xpt{11}} % 10 point on 11 +\def\small{\@setfontsize\small\@ixpt{10}} % 9 point on 10 +\def\footnotesize{\@setfontsize\footnotesize\@ixpt{10}} % 9 point on 10 +\def\scriptsize{\@setfontsize\scriptsize\@viipt{10}} % 7 point on 8 +\def\tiny{\@setfontsize\tiny\@vipt{7}} % 6 point on 7 +\def\large{\@setfontsize\large\@xipt{12}} % 11 point on 12 +\def\Large{\@setfontsize\Large\@xiipt{14}} % 12 point on 14 +\def\LARGE{\@setfontsize\LARGE\@xivpt{16}} % 14 point on 16 +\def\huge{\@setfontsize\huge\@xviipt{20}} % 17 point on 20 +\def\Huge{\@setfontsize\Huge\@xxpt{23}} % 20 point on 23 + +\RequirePackage{xcolor} % Needed for the reproducibility checklist +\def\isChecklistMainFile{} % Signal the checklist that it's included from the main file + +\AtBeginDocument{% + \@ifpackageloaded{natbib}% + {% + % When natbib is in use, set the proper style and fix a few things + \let\cite\citep + \let\shortcite\citeyearpar + \setcitestyle{aysep={}} + \setlength\bibhang{0pt} + \bibliographystyle{aaai2026} + }{}% + \@ifpackageloaded{hyperref}% + {% + \PackageError{aaai}{You must not use hyperref in AAAI papers.}{You (or one of the packages you imported) are importing the hyperref package, which is forbidden in AAAI papers. You must remove it from the paper to proceed.} + }{}% + \@ifpackageloaded{bbm}% + {% + \PackageError{aaai}{You must not use bbm package in AAAI papers because it introduces Type 3 fonts which are forbidden.}{See https://tex.stackexchange.com/questions/479160/a-replacement-to-mathbbm1-with-type-1-fonts for possible alternatives.} + }{}% + \@ifpackageloaded{authblk}% + {% + \PackageError{aaai}{Package authblk is forbbidden.}{Package authblk is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{balance}% + {% + \PackageError{aaai}{Package balance is forbbidden.}{Package balance is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{CJK}% + {% + \PackageError{aaai}{Package CJK is forbbidden.}{Package CJK is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{flushend}% + {% + \PackageError{aaai}{Package flushend is forbbidden.}{Package flushend is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{fontenc}% + {% + \PackageError{aaai}{Package fontenc is forbbidden.}{Package fontenc is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{fullpage}% + {% + \PackageError{aaai}{Package fullpage is forbbidden.}{Package fullpage is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{geometry}% + {% + \PackageError{aaai}{Package geometry is forbbidden.}{Package geometry is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{grffile}% + {% + \PackageError{aaai}{Package grffile is forbbidden.}{Package grffile is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{navigator}% + {% + \PackageError{aaai}{Package navigator is forbbidden.}{Package navigator is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{savetrees}% + {% + \PackageError{aaai}{Package savetrees is forbbidden.}{Package savetrees is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{setspace}% + {% + \PackageError{aaai}{Package setspace is forbbidden.}{Package setspace is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{stfloats}% + {% + \PackageError{aaai}{Package stfloats is forbbidden.}{Package stfloats is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{tabu}% + {% + \PackageError{aaai}{Package tabu is forbbidden.}{Package tabu is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{titlesec}% + {% + \PackageError{aaai}{Package titlesec is forbbidden.}{Package titlesec is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{tocbibind}% + {% + \PackageError{aaai}{Package tocbibind is forbbidden.}{Package tocbibind is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{ulem}% + {% + \PackageError{aaai}{Package ulem is forbbidden.}{Package ulem is forbbiden. You must find an alternative.} + }{}% + \@ifpackageloaded{wrapfig}% + {% + \PackageError{aaai}{Package wrapfig is forbbidden.}{Package wrapfig is forbbiden. You must find an alternative.} + }{}% +} + +\let\endthebibliography=\endlist diff --git a/final_report/build_report.py b/final_report/build_report.py new file mode 100644 index 000000000..a2a0565f9 --- /dev/null +++ b/final_report/build_report.py @@ -0,0 +1,777 @@ +from __future__ import annotations + +import io +import math +import subprocess +import textwrap +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd + + +ROOT = Path(__file__).resolve().parents[1] +OUT = Path(__file__).resolve().parent +SOURCE_REF = "origin/refactor/tabm-integrated-modules" +SUMMARY_PATH = "tabm_integrated/paper/exp/final_integrated_summary.csv" +REPORT_PATH = "tabm_integrated/FINAL_EXPERIMENT_REPORT.md" + +DATASET_ORDER = [ + "sberbank-housing", + "ecom-offers", + "homesite-insurance", + "cooking-time", + "delivery-eta", +] + +DATASET_LABEL = { + "sberbank-housing": "Sberbank", + "ecom-offers": "Ecom", + "homesite-insurance": "Homesite", + "cooking-time": "Cooking", + "delivery-eta": "Delivery", +} + +TABLE_DATA_LABEL = { + "sberbank-housing": "Sb", + "ecom-offers": "Ec", + "homesite-insurance": "Hs", + "cooking-time": "Ck", + "delivery-eta": "Dl", +} + +VARIANT_LABEL = { + "best_rla_only": "RLA", + "best_esam_only": "ESAM", + "best_mfb_only": "MFB", + "best_cf_fisd_only": "CF-FISD", + "best_combined": "Combined", +} + +STATUS_SCORE = { + "clear_win": 2, + "weak_win": 1, + "tie": 0, + "loss": -1, +} + +STATUS_LABEL = { + "clear_win": "Clear win", + "weak_win": "Weak win", + "tie": "Tie", + "loss": "Loss", + "baseline": "Baseline", +} + +COMBO_LABEL = { + "sb_comb_rla_mfb_cf_r1_n1em05_k0p975_l0p001": "R+M+CF", + "mfb_cf_fisd": "M+CF", + "rla_esam": "R+E", + "deliv_comb_rla_cf": "R+CF", +} + + +def git_show(path: str) -> str: + return subprocess.check_output( + ["git", "show", f"{SOURCE_REF}:{path}"], + cwd=ROOT, + text=True, + ) + + +def load_summary() -> pd.DataFrame: + csv_text = git_show(SUMMARY_PATH) + (OUT / "source_final_integrated_summary.csv").write_text(csv_text) + return pd.read_csv(io.StringIO(csv_text)) + + +def fmt_num(x: float, digits: int = 6) -> str: + if x is None or not math.isfinite(float(x)): + return "--" + x = float(x) + if abs(x) >= 10: + return f"{x:.3f}" + if abs(x) >= 1: + return f"{x:.4f}" + return f"{x:.6f}" + + +def fmt_pm(mean: float, std: float) -> str: + return f"{fmt_num(mean)} $\\pm$ {fmt_num(std)}" + + +def fmt_compact(x: float) -> str: + if x is None or not math.isfinite(float(x)): + return "--" + return f"{float(x):.4f}" + + +def fmt_pct(x: float) -> str: + if x is None or not math.isfinite(float(x)): + return "--" + sign = "+" if x > 0 else "" + return f"{sign}{x:.3f}\\%" + + +def latex_escape(value: object) -> str: + s = str(value) + return ( + s.replace("\\", "\\textbackslash{}") + .replace("&", "\\&") + .replace("%", "\\%") + .replace("$", "\\$") + .replace("#", "\\#") + .replace("_", "\\_") + .replace("{", "\\{") + .replace("}", "\\}") + .replace("~", "\\textasciitilde{}") + .replace("^", "\\textasciicircum{}") + ) + + +def short_inference(mode: str) -> str: + return { + "mean": "mean", + "best-head": "best", + "greedy-heads": "greedy", + }.get(mode, mode) + + +def default_delta_pct(row: pd.Series, base_rows: pd.DataFrame) -> float: + base = base_rows.loc[row["dataset"]] + if row["direction"] == "lower": + delta = float(base["mean"]) - float(row["mean"]) + else: + delta = float(row["mean"]) - float(base["mean"]) + return 100.0 * delta / abs(float(base["mean"])) + + +def make_combined_plot(df: pd.DataFrame) -> None: + base = df[df["variant"] == "baseline_plr"].set_index("dataset") + combined = df[df["variant"] == "best_combined"].set_index("dataset").loc[DATASET_ORDER].reset_index() + matched = combined["percent_delta"].astype(float).to_numpy() + default = np.array([default_delta_pct(row, base) for _, row in combined.iterrows()]) + + x = np.arange(len(DATASET_ORDER)) + width = 0.36 + fig, ax = plt.subplots(figsize=(7.2, 3.0)) + ax.axhline(0, color="#2b2b2b", linewidth=0.8) + ax.bar(x - width / 2, matched, width, label="Matched inference", color="#234f68") + ax.bar(x + width / 2, default, width, label="Default mean baseline", color="#c47d30") + ax.set_xticks(x) + ax.set_xticklabels([DATASET_LABEL[d] for d in DATASET_ORDER], rotation=20, ha="right") + ax.set_ylabel("Improvement over baseline (%)") + ax.set_title("Combined method: matched and default-mean comparisons") + ax.legend(frameon=False, fontsize=8) + ax.grid(axis="y", alpha=0.25) + fig.tight_layout() + fig.savefig(OUT / "fig_combined_comparison.pdf") + fig.savefig(OUT / "fig_combined_comparison.png", dpi=220) + plt.close(fig) + + +def make_status_heatmap(df: pd.DataFrame) -> None: + variants = list(VARIANT_LABEL) + matrix = [] + for variant in variants: + row = [] + rows = df[df["variant"] == variant].set_index("dataset") + for ds in DATASET_ORDER: + row.append(STATUS_SCORE[str(rows.loc[ds, "status"])]) + matrix.append(row) + matrix = np.array(matrix, dtype=float) + + fig, ax = plt.subplots(figsize=(7.2, 2.7)) + cmap = plt.matplotlib.colors.ListedColormap(["#b94b45", "#d7d7d7", "#e0b44d", "#3c7d59"]) + bounds = [-1.5, -0.5, 0.5, 1.5, 2.5] + norm = plt.matplotlib.colors.BoundaryNorm(bounds, cmap.N) + ax.imshow(matrix, cmap=cmap, norm=norm, aspect="auto") + ax.set_xticks(np.arange(len(DATASET_ORDER))) + ax.set_xticklabels([DATASET_LABEL[d] for d in DATASET_ORDER], rotation=20, ha="right") + ax.set_yticks(np.arange(len(variants))) + ax.set_yticklabels([VARIANT_LABEL[v] for v in variants]) + for i, variant in enumerate(variants): + rows = df[df["variant"] == variant].set_index("dataset") + for j, ds in enumerate(DATASET_ORDER): + status = str(rows.loc[ds, "status"]) + txt = {"clear_win": "C", "weak_win": "W", "tie": "T", "loss": "L"}[status] + ax.text(j, i, txt, ha="center", va="center", fontsize=9, color="black") + ax.set_title("Validation-selected module outcomes vs matched baselines") + ax.tick_params(length=0) + fig.tight_layout() + fig.savefig(OUT / "fig_module_heatmap.pdf") + fig.savefig(OUT / "fig_module_heatmap.png", dpi=220) + plt.close(fig) + + +def make_win_count_plot(df: pd.DataFrame) -> None: + variants = list(VARIANT_LABEL) + counts = [] + for variant in variants: + rows = df[df["variant"] == variant] + counts.append(int(rows["status"].isin(["weak_win", "clear_win"]).sum())) + fig, ax = plt.subplots(figsize=(5.8, 2.5)) + ax.bar([VARIANT_LABEL[v] for v in variants], counts, color="#4f6f52") + ax.set_ylim(0, 5.4) + ax.set_ylabel("Datasets improved (of 5)") + ax.set_title("Individual modules are complementary; combined reaches 5/5") + for idx, c in enumerate(counts): + ax.text(idx, c + 0.08, str(c), ha="center", va="bottom", fontsize=9) + ax.grid(axis="y", alpha=0.25) + fig.tight_layout() + fig.savefig(OUT / "fig_win_counts.pdf") + fig.savefig(OUT / "fig_win_counts.png", dpi=220) + plt.close(fig) + + +def make_delta_heatmap(df: pd.DataFrame) -> None: + variants = list(VARIANT_LABEL) + matrix = [] + for variant in variants: + rows = df[df["variant"] == variant].set_index("dataset") + matrix.append([float(rows.loc[ds, "percent_delta"]) for ds in DATASET_ORDER]) + matrix = np.array(matrix) + vmax = max(2.2, float(np.nanmax(np.abs(matrix)))) + fig, ax = plt.subplots(figsize=(7.4, 2.9)) + im = ax.imshow(matrix, cmap="RdYlGn", vmin=-vmax, vmax=vmax, aspect="auto") + ax.set_xticks(np.arange(len(DATASET_ORDER))) + ax.set_xticklabels([DATASET_LABEL[d] for d in DATASET_ORDER], rotation=20, ha="right") + ax.set_yticks(np.arange(len(variants))) + ax.set_yticklabels([VARIANT_LABEL[v] for v in variants]) + for i in range(matrix.shape[0]): + for j in range(matrix.shape[1]): + ax.text(j, i, f"{matrix[i, j]:+.2f}", ha="center", va="center", fontsize=8) + ax.set_title("Matched-inference improvement (%) by module") + ax.tick_params(length=0) + cb = fig.colorbar(im, ax=ax, fraction=0.035, pad=0.02) + cb.set_label("Improvement (%)", fontsize=8) + fig.tight_layout() + fig.savefig(OUT / "fig_module_delta_heatmap.pdf") + fig.savefig(OUT / "fig_module_delta_heatmap.png", dpi=220) + plt.close(fig) + + +def make_protocol_plot() -> None: + fig, ax = plt.subplots(figsize=(7.4, 2.2)) + ax.axis("off") + boxes = [ + ("TabM paper\nPLR configs", 0.08, "#d9e7ef"), + ("Reproduced\nTabM baseline", 0.29, "#e7ead7"), + ("Member-level\nmodules", 0.50, "#efe1cf"), + ("Validation-only\nselection", 0.71, "#ead8e9"), + ("Matched + default\ncomparisons", 0.90, "#dce6da"), + ] + y = 0.52 + for label, x, color in boxes: + ax.text( + x, + y, + label, + ha="center", + va="center", + fontsize=9, + bbox=dict(boxstyle="round,pad=0.35", facecolor=color, edgecolor="#333333", linewidth=0.9), + transform=ax.transAxes, + ) + for (_, x0, _), (_, x1, _) in zip(boxes[:-1], boxes[1:]): + ax.annotate( + "", + xy=(x1 - 0.08, y), + xytext=(x0 + 0.08, y), + arrowprops=dict(arrowstyle="->", linewidth=1.2, color="#333333"), + xycoords=ax.transAxes, + textcoords=ax.transAxes, + ) + ax.text( + 0.5, + 0.13, + "No data-policy changes, no paper-baseline retuning, final rows use three seeds.", + ha="center", + va="center", + fontsize=8, + color="#333333", + transform=ax.transAxes, + ) + fig.tight_layout() + fig.savefig(OUT / "fig_protocol_bridge.pdf") + fig.savefig(OUT / "fig_protocol_bridge.png", dpi=220) + plt.close(fig) + + +def make_method_diagram() -> None: + fig, ax = plt.subplots(figsize=(7.4, 3.1)) + ax.axis("off") + + boxes = [ + ("Input\nfeatures", 0.08, 0.55, "#e9ecef"), + ("MFB\nmember masks", 0.25, 0.55, "#f1dfc9"), + ("PLR\nembeddings", 0.42, 0.55, "#dce8f2"), + ("Shared\nMLP", 0.59, 0.55, "#e4ebd4"), + ("Member\nadapters", 0.76, 0.55, "#ead8e8"), + ("Predictions\nmean/best/greedy", 0.92, 0.55, "#dbe7dc"), + ] + for label, x, y, color in boxes: + ax.text( + x, + y, + label, + ha="center", + va="center", + fontsize=9, + bbox=dict(boxstyle="round,pad=0.38", facecolor=color, edgecolor="#333333", linewidth=0.9), + transform=ax.transAxes, + ) + for (_, x0, y0, _), (_, x1, y1, _) in zip(boxes[:-1], boxes[1:]): + ax.annotate( + "", + xy=(x1 - 0.065, y1), + xytext=(x0 + 0.065, y0), + arrowprops=dict(arrowstyle="->", linewidth=1.2, color="#333333"), + xycoords=ax.transAxes, + textcoords=ax.transAxes, + ) + + annotations = [ + ("ESAM perturbs\nadapter parameters\nduring training", 0.70, 0.20, 0.74, 0.44), + ("RLA changes\nadapter rank", 0.77, 0.86, 0.77, 0.66), + ("CF-FISD aligns\nadapter saliency", 0.90, 0.86, 0.79, 0.66), + ] + for text, tx, ty, px, py in annotations: + ax.annotate( + text, + xy=(px, py), + xytext=(tx, ty), + ha="center", + va="center", + fontsize=8, + bbox=dict(boxstyle="round,pad=0.25", facecolor="#ffffff", edgecolor="#555555", linewidth=0.8), + arrowprops=dict(arrowstyle="->", linewidth=1.0, color="#555555"), + xycoords=ax.transAxes, + textcoords=ax.transAxes, + ) + ax.text( + 0.5, + 0.04, + "All modules are member-level changes around the official TabM-PLR pipeline.", + ha="center", + va="center", + fontsize=8, + transform=ax.transAxes, + ) + fig.tight_layout() + fig.savefig(OUT / "fig_method_diagram.pdf") + fig.savefig(OUT / "fig_method_diagram.png", dpi=220) + plt.close(fig) + + +def combined_table(df: pd.DataFrame) -> str: + base = df[df["variant"] == "baseline_plr"].set_index("dataset") + combined = df[df["variant"] == "best_combined"].set_index("dataset").loc[DATASET_ORDER].reset_index() + lines = [ + "\\begin{table}[!htbp]", + "\\centering", + "\\setlength{\\tabcolsep}{2pt}", + "\\footnotesize", + "\\begin{tabular}{lllccc}", + "\\hline", + "Data & Combo & Inf. & $\\Delta_m$ & $\\Delta_\\mu$ & S \\\\", + "\\hline", + ] + for _, row in combined.iterrows(): + mean_delta = default_delta_pct(row, base) + combo = COMBO_LABEL.get(str(row["source_variant"]), str(row["source_variant"])) + data_metric = f"{TABLE_DATA_LABEL[row['dataset']]}/{row['metric']}" + status_code = {"clear_win": "C", "weak_win": "W", "tie": "T", "loss": "L"}[str(row["status"])] + lines.append( + f"{latex_escape(data_metric)} & " + f"{latex_escape(combo)} & " + f"{latex_escape(short_inference(str(row['inference_mode'])))} & " + f"{fmt_pct(row['percent_delta'])} & " + f"{fmt_pct(mean_delta)} & " + f"{status_code} \\\\" + ) + lines += [ + "\\hline", + "\\end{tabular}", + "\\caption{Final validation-selected combined results. Data includes the metric. Combo abbreviations: R = RLA, E = ESAM, M = MFB, CF = CF-FISD. $\\Delta_m$ is matched-inference improvement; $\\Delta_\\mu$ is improvement over default mean-inference TabM-PLR. S: C = clear win, W = weak win. Mean and standard deviation values are in the released CSV artifact.}", + "\\label{tab:combined}", + "\\end{table}", + ] + return "\n".join(lines) + + +def ablation_summary_table() -> str: + rows = [ + ("Baseline fidelity", "Paper-config TabM-PLR is the reference row; we do not lower or simplify the baseline."), + ("RLA capacity", "Rank, initialization, noise, freeze, and inference sweeps show capacity helps Sb. and Dl., but not every dataset."), + ("ESAM sharpness", "Rho and adapter-only sweeps show ESAM helps Ec., Hs., and Ck."), + ("MFB diversity", "Member mask and keep-rate sweeps show feature diversity helps Ec., Ck., and Dl."), + ("CF-FISD saliency", "Teacher-alignment and lambda sweeps help Sb. and Hs.; Dl. is neutral under matched inference."), + ("Combined subsets", "Compatible module subsets are validation-selected per dataset; the selected combinations improve 5/5 matched."), + ("Inference audit", "Mean, best-head, and greedy-head baselines are compared separately to avoid inflated claims."), + ("Rejected screens", "Auxiliary objectives and k-scaling screens are not used in the final selected rows."), + ] + lines = [ + "\\begin{table}[!htbp]", + "\\centering", + "\\footnotesize", + "\\setlength{\\tabcolsep}{2pt}", + "\\begin{tabular}{p{0.27\\columnwidth}p{0.65\\columnwidth}}", + "\\hline", + "Ablation & What it showed \\\\", + "\\hline", + ] + for name, takeaway in rows: + lines.append(f"{latex_escape(name)} & {latex_escape(takeaway)} \\\\") + lines += [ + "\\hline", + "\\end{tabular}", + "\\caption{Ablation coverage in the main report. The raw grids contain more individual trials; the table lists the ablation families that affect the final conclusion.}", + "\\label{tab:ablations}", + "\\end{table}", + ] + return "\n".join(lines) + + +def paper_alignment_table() -> str: + rows = [ + ("Paper target", "TabM, ICLR 2025"), + ("Baseline", "Official per-dataset TabM-PLR configs"), + ("Data", "Five available TabReD datasets"), + ("Metrics", "RMSE for regression, AUROC for binary tasks"), + ("Kept fixed", "Preprocessing, data policy, tuned optimizer, $k$, patience"), + ("Changed", "Only module-specific adapter/loss/masking flags"), + ("Selection", "Validation-only; report matched and default-mean baselines"), + ] + lines = [ + "\\begin{table}[!htbp]", + "\\centering", + "\\small", + "\\begin{tabular}{p{0.23\\columnwidth}p{0.65\\columnwidth}}", + "\\hline", + "Item & Protocol \\\\", + "\\hline", + ] + for item, protocol in rows: + lines.append(f"{latex_escape(item)} & {protocol} \\\\") + lines += [ + "\\hline", + "\\end{tabular}", + "\\caption{Comparison protocol against the TabM paper implementation. The report compares against a reproduced paper-config TabM-PLR baseline, not a simplified local baseline.}", + "\\label{tab:paper_alignment}", + "\\end{table}", + ] + return "\n".join(lines) + + +def baseline_values_table(df: pd.DataFrame) -> str: + base = df[df["variant"] == "baseline_plr"].set_index("dataset") + combined = df[df["variant"] == "best_combined"].set_index("dataset").loc[DATASET_ORDER].reset_index() + lines = [ + "\\begin{table}[!htbp]", + "\\centering", + "\\footnotesize", + "\\setlength{\\tabcolsep}{2pt}", + "\\begin{tabular}{lcccc}", + "\\hline", + "Data & Default & Matched & Ours & $\\Delta_m$ \\\\", + "\\hline", + ] + for _, row in combined.iterrows(): + ds = row["dataset"] + data_metric = f"{TABLE_DATA_LABEL[ds]}/{row['metric']}" + lines.append( + f"{latex_escape(data_metric)} & " + f"{fmt_compact(base.loc[ds, 'mean'])} & " + f"{fmt_compact(row['matched_baseline_mean'])} & " + f"{fmt_compact(row['mean'])} & " + f"{fmt_pct(row['percent_delta'])} \\\\" + ) + lines += [ + "\\hline", + "\\end{tabular}", + "\\caption{Paper-config baseline comparison. Default is the reproduced default mean-inference TabM-PLR baseline. Matched is TabM-PLR under the same selected inference mode as Ours.}", + "\\label{tab:baseline_values}", + "\\end{table}", + ] + return "\n".join(lines) + + +def module_table(df: pd.DataFrame) -> str: + variants = list(VARIANT_LABEL) + code = {"clear_win": "C", "weak_win": "W", "tie": "T", "loss": "L"} + lines = [ + "\\begin{table}[!htbp]", + "\\centering", + "\\small", + "\\begin{tabular}{lccccc}", + "\\hline", + "Method & Sb. & Ec. & Hs. & Ck. & Dl. \\\\", + "\\hline", + ] + for variant in variants: + rows = df[df["variant"] == variant].set_index("dataset") + vals = [code[str(rows.loc[ds, "status"])] for ds in DATASET_ORDER] + lines.append(f"{latex_escape(VARIANT_LABEL[variant])} & " + " & ".join(vals) + " \\\\") + lines += [ + "\\hline", + "\\end{tabular}", + "\\caption{Matched-baseline outcome matrix. C = clear win, W = weak win, T = tie, L = loss. Dataset abbreviations follow Table~\\ref{tab:combined}.}", + "\\label{tab:matrix}", + "\\end{table}", + ] + return "\n".join(lines) + + +def make_tex(df: pd.DataFrame) -> str: + combined = combined_table(df) + baseline_values = baseline_values_table(df) + ablations = ablation_summary_table() + modules = module_table(df) + report_md = git_show(REPORT_PATH) + (OUT / "source_final_experiment_report.md").write_text(report_md) + + return textwrap.dedent( + rf""" + % Auto-generated by final_report/build_report.py. + \documentclass[letterpaper]{{article}} + \usepackage[draft]{{aaai2026}} + \usepackage{{times}} + \usepackage{{helvet}} + \usepackage{{courier}} + \usepackage[hyphens]{{url}} + \usepackage{{graphicx}} + \urlstyle{{rm}} + \def\UrlFont{{\rm}} + \usepackage{{natbib}} + \usepackage{{caption}} + \frenchspacing + \setlength{{\pdfpagewidth}}{{8.5in}} + \setlength{{\pdfpageheight}}{{11in}} + \providecommand{{\pdfinfo}}[1]{{}} + \pdfinfo{{/TemplateVersion (2026.1)}} + \setcounter{{secnumdepth}}{{0}} + + \title{{Complementary Member-Level Regularization for TabM on TabReD}} + \author{{Prithvi Nishal (G2504965H), Pentamsetty Sai Harshita (G2503340A),\\ + Nath Simerjit Kaur (G2507742D), Abhipray Chavan (G2504327J)}} + \affiliations{{Nanyang Technological University}} + + \begin{{document}} + \maketitle + + \begin{{abstract}} + TabM is a recent tabular deep learning baseline built around parameter-efficient ensembling. We test four concrete member-level questions on five TabReD datasets: whether TabM benefits from more adapter capacity, flatter adapter optima, fixed member-level feature diversity, or structured adapter saliency. Our implementation keeps the official TabM-PLR per-dataset pipeline and adds four corresponding modifications: rank-adaptive low-rank adapters (RLA), adapter-focused sharpness-aware optimization (ESAM), member-fixed feature bagging (MFB), and teacher-importance distillation over adapter feature saliency (CF-FISD). Individually, no module dominates all datasets. Under validation-selected matched inference, a per-dataset set of compatible module combinations improves over the corresponding TabM-PLR baseline on all five datasets. The result is intentionally scoped: default mean-inference TabM-PLR remains better on some datasets, so the main finding is complementarity under matched inference, not a replacement for the paper baseline. + \end{{abstract}} + + \section{{Introduction}} + Tabular data remains a difficult setting for deep learning because tree models and well-tuned MLPs are hard baselines. TabM~\citep{{gorishniy2025tabm}} addresses this by producing multiple predictions from one parameter-efficient MLP ensemble based on BatchEnsemble-style member adapters~\citep{{wen2020batchensemble}}. The paper shows that the members are weak individually but useful collectively; this makes the internal member structure a concrete part of the model to test. Our project asks whether that structure can be improved by shaping capacity, optimization, feature exposure, and feature saliency at the member level. + + We use the project option of improving a post-2019 conference paper with its public codebase. The target paper is TabM, an ICLR 2025 paper, and the benchmark is TabReD~\citep{{rubachev2024tabred}}, a tabular benchmark with realistic splits and heterogeneous tasks. PLR in this report means the PiecewiseLinearEmbeddings numerical embedding setup used by the TabM paper configs. The contribution is not a new dataset or a lowered baseline. Instead, the report evaluates four modifications on top of the same TabM-PLR per-dataset configs. + + The main result has a narrow scope. The validation-selected combined method improves over matched-inference TabM-PLR on all five selected TabReD datasets. This is not the same as beating the default mean-inference TabM baseline everywhere. We report both comparisons to avoid artificially lowering the baseline. + + \section{{Relationship to the TabM Paper}} + The paper comparison in this report is against the official TabM-PLR implementation path. We do not replace TabM with a simplified MLP, retune the baseline downward, or change the per-dataset data policies. The official paper pipeline uses TabM with PiecewiseLinearEmbeddings, tuned optimizer settings, dataset-specific policies, and $k=32$ members for these TabReD-style experiments. Our code keeps that pipeline and adds module flags around the member adapters. + + Table~\ref{{tab:paper_alignment}} summarizes the alignment. The important distinction is that the released paper used a broader evaluation budget, while our final confirmation run uses three seeds for every final row. Therefore, our result should be read as a controlled project reproduction and extension of the TabM paper configuration, not as a claim that we have replaced the full published leaderboard. + + {paper_alignment_table()} + + \begin{{figure}}[!htbp] + \centering + \includegraphics[width=\columnwidth]{{fig_protocol_bridge.pdf}} + \caption{{How the experiment connects the TabM paper implementation to our modified system.}} + \label{{fig:protocol}} + \end{{figure}} + + \section{{Methods}} + \textbf{{TabM baseline.}} TabM trains an MLP backbone that emits $k$ member predictions using parameter-efficient per-member multiplicative adapters. At inference, the default prediction is the mean over members, while best-head and greedy-head modes select heads or subsets using validation scores. Our baseline is the official TabM-PLR configuration for each dataset: tuned optimizer, architecture, embeddings, data policy, $k$, patience, and dataset-specific preprocessing are preserved. + + \textbf{{RLA.}} Rank-adaptive low-rank adapters generalize the rank-1 member adapter path to multiple low-rank paths while preserving the original base path. The intended effect is to increase member-specific capacity without turning TabM into a fully separate ensemble. Empirically, this helped most when the additional paths were small and base-preserving; aggressive extra capacity could overfit or destabilize the already competitive PLR baseline. + + \textbf{{ESAM.}} ESAM applies sharpness-aware optimization~\citep{{foret2021sam}} to the member-specific adapter parameters. This targets per-member overfitting and encourages flatter adapter solutions while leaving the common TabM training pipeline intact. We restrict the perturbation to adapter-heavy parameter groups so that ESAM changes the ensemble members more than the shared backbone. + + \textbf{{MFB.}} Member-fixed feature bagging assigns deterministic feature masks to TabM members. This directly increases input-level diversity among members while retaining the shared backbone. Unlike stochastic dropout, the masks are member-specific and stable, so each member learns a consistent view of the feature space. + + \textbf{{CF-FISD.}} CF-FISD aligns member adapter feature saliency with feature-importance profiles from external tabular teachers. The implemented loss groups TabM members by teacher family and penalizes mismatch between adapter saliency and teacher feature importance. This gives a structured signal to adapter weights without feeding teacher predictions into the final model. + + These four methods target different questions about TabM's members: adapter capacity, optimization sharpness, input-level diversity, and feature-saliency alignment. The combined system is validation-selected per dataset from compatible module combinations. It is therefore a validation-selected model family, not a single fixed all-four architecture. + + Conceptually, the modules act at four different points of the same TabM member pipeline. If $f_i(x)$ is the prediction of member $i$, RLA changes the capacity of the member adapter producing $f_i$, ESAM changes the local optimization landscape of those adapter parameters, MFB changes the member-specific input view of $x$, and CF-FISD regularizes the adapter saliency pattern. The design goal is complementarity rather than one universal knob. + + \begin{{figure}}[!htbp] + \centering + \includegraphics[width=\columnwidth]{{fig_method_diagram.pdf}} + \caption{{Where each modification acts in the TabM-PLR pipeline. The diagram is intentionally limited to the mechanisms used in the final integrated code.}} + \label{{fig:method}} + \end{{figure}} + + \section{{Implementation Details}} + The integrated code keeps a single TabM training entry point and exposes each modification through explicit configuration fields. This is important because the baseline and modified runs share the same data loader, metric computation, early stopping, and evaluation code. When all module flags are disabled, the run path is the paper-config TabM-PLR baseline. When a module is enabled, the change is localized to adapter construction, optimizer behavior, feature masking, or the auxiliary CF-FISD loss. + + RLA is implemented in the adapter construction path. The base-preserving variants keep the original rank-1 behavior as one path and add low-rank residual paths with small initialization noise. This design was chosen after early sweeps showed that simply increasing adapter rank could hurt on saturated datasets. ESAM is implemented in the training step rather than as a separate model: it perturbs selected adapter parameters, evaluates the sharpness-aware objective, and restores the parameters before the optimizer update. MFB is implemented before the shared MLP blocks by applying member-indexed masks over feature groups. CF-FISD is implemented as an auxiliary loss over the first adapter saliency, with teacher feature-importance vectors loaded from the bundled teacher artifacts. + + The combined rows are not a single all-modules-on run. They are validation-selected compositions from compatible module subsets. This matters because some modules conflict on specific datasets: for example, feature bagging can help one dataset while hurting another, and extra adapter capacity can be useful for one target but noisy for another. The per-dataset selection rule is therefore part of the method. The report names this explicitly so that the reader does not confuse the final 5/5 row with a single fixed architecture. + + The implementation also records enough metadata to audit the comparisons. Each final row contains the selected source variant, metric direction, inference mode, matched baseline mean, default mean baseline, final mean and standard deviation, and seed count. Without those fields, the same numerical table could accidentally compare a best-head result to a mean-inference baseline, which would overstate the result. The final report is generated from that CSV rather than from manually typed values. + + \section{{Experimental Protocol}} + We evaluate five TabReD datasets: sberbank-housing, ecom-offers, homesite-insurance, cooking-time, and delivery-eta. Regression datasets use RMSE, where lower is better. Binary classification datasets use AUROC, where higher is better. Using both metrics is expected because TabReD mixes regression and binary classification tasks. + + Every final reported row uses three seeds. Model selection is performed using validation metrics only. For a fair comparison with validation-selected inference modes, the main delta compares each selected variant to TabM-PLR evaluated with the same inference mode. We also report the delta against the default mean-inference baseline, because it is the most conservative view of whether the method improves the usual TabM deployment. The final matrix contains 30 rows: one TabM-PLR baseline plus five method rows for each dataset. + + The protocol deliberately separates three questions. First, does an individual module help under a matched inference mode? Second, do the modules help different datasets, making them complementary? Third, does a validation-selected combined method improve over a correspondingly selected TabM baseline? This separation is necessary because a single sign-based table can hide the fact that an improvement depends on the inference mode. + + \section{{Ablation Study}} + We ran more ablations than can fit as raw tables in the main report. The main paper therefore reports ablation families, not every single failed setting. This is deliberate: the reader should see the logic of the search and the evidence behind the final choices without having to parse hundreds of near-duplicate runs. Table~\ref{{tab:ablations}} lists the ablations that changed the final interpretation. + + {ablations} + + The ablations answer a specific question rather than just searching for a better number. RLA tests whether TabM's rank-1 member adapter is a capacity bottleneck. ESAM tests whether the member adapters overfit sharply. MFB tests whether fixed member-specific feature exposure improves diversity. CF-FISD tests whether teacher feature-importance structure can guide adapter saliency. The combined-subset ablation then tests whether the datasets helped by each module are complementary. This avoids the weak argument ``we tried many things and one worked''; the evidence is organized around concrete properties of TabM's member ensemble. + + \section{{Results}} + Table~\ref{{tab:combined}} shows the final validation-selected combined method. The selected combinations differ by dataset: RLA+MFB+CF-FISD for sberbank-housing, MFB+CF-FISD for ecom-offers and homesite-insurance, RLA+ESAM for cooking-time, and RLA+CF-FISD for delivery-eta. These selected combinations improve all five datasets under matched inference: +2.047\% on sberbank-housing, +0.806\% AUROC on ecom-offers, +0.018\% AUROC on homesite-insurance, +0.021\% on cooking-time, and +0.423\% on delivery-eta. The wins are small on several datasets, which is expected because TabM-PLR is already a competitive baseline. + + {combined} + + Table~\ref{{tab:baseline_values}} makes the paper-config comparison explicit. The first numeric baseline is the reproduced default mean-inference TabM-PLR row, which corresponds to the usual paper deployment mode. The matched baseline is the same TabM-PLR model evaluated with the selected inference mode. This distinction matters because the matched baseline can be weaker or stronger than default mean inference depending on the dataset. + + {baseline_values} + + The default mean comparison is more conservative. It remains positive on ecom-offers, homesite-insurance, and cooking-time, but negative on sberbank-housing and delivery-eta. This means the safe claim is not that the method improves over default TabM in every deployment mode. The safe claim is that validation-selected module composition improves the corresponding validation-selected TabM inference baseline on every dataset. + + \begin{{figure}}[!htbp] + \centering + \includegraphics[width=\columnwidth]{{fig_combined_comparison.pdf}} + \caption{{Combined method improvements under the matched-inference protocol and the conservative default mean-inference comparison.}} + \label{{fig:combined}} + \end{{figure}} + + Figure~\ref{{fig:combined}} is the central paper-baseline comparison. The blue bars compare the selected combined method to the TabM-PLR baseline under the same inference mode. The orange bars compare the same selected method to the default mean-inference TabM-PLR baseline. This is why the paper claim is not overstated: sberbank-housing and delivery-eta are still worse than default mean TabM, even though they are positive under matched best-head inference. + + \begin{{figure}}[!htbp] + \centering + \includegraphics[width=\columnwidth]{{fig_module_heatmap.pdf}} + \caption{{Outcome heatmap for individual modules and the final combined system.}} + \label{{fig:heatmap}} + \end{{figure}} + + {modules} + + Table~\ref{{tab:matrix}} and Figure~\ref{{fig:heatmap}} show why combining modules matters. RLA helps sberbank-housing and delivery-eta, ESAM helps ecom-offers, homesite-insurance, and cooking-time, MFB helps ecom-offers, cooking-time, and delivery-eta, and CF-FISD helps sberbank-housing and homesite-insurance. No individual modification is universal, but the error patterns are complementary. + + \begin{{figure}}[!htbp] + \centering + \includegraphics[width=\columnwidth]{{fig_module_delta_heatmap.pdf}} + \caption{{Magnitude of matched-inference improvements by module. Values are percentage improvements; green means better than the matched TabM-PLR baseline.}} + \label{{fig:delta_heatmap}} + \end{{figure}} + + Figure~\ref{{fig:delta_heatmap}} adds magnitude to the win/loss matrix. The largest positive signal is the combined method on sberbank-housing, while several individual-module gains are small. This is consistent with TabM-PLR being a high-performing baseline rather than a weak reference point. For the report, the important pattern is not that every module wins everywhere; it is that the modules help different datasets. + + \begin{{figure}}[!htbp] + \centering + \includegraphics[width=\columnwidth]{{fig_win_counts.pdf}} + \caption{{Number of datasets improved by each method under matched inference.}} + \label{{fig:wins}} + \end{{figure}} + + Figure~\ref{{fig:wins}} summarizes this complementarity. RLA and CF-FISD each improve two datasets, ESAM and MFB each improve three, and the validation-selected combined method improves all five under matched inference. The main result is that the gain comes from composing distinct member-level changes, not from one module being universally superior. + + \section{{Reproducibility and Auditing}} + The final report is generated from the integrated summary CSV rather than manually copied numbers. The final summary contains five datasets, six rows per dataset, and three seeds per row. Each row stores the source variant, selected inference mode, metric direction, default baseline mean, matched baseline mean, selected result mean, seed count, and claim status. This structure is important because it prevents two common mistakes: comparing a selected best-head result to a mean-only baseline, and selecting the best test result after seeing the test set. + + We also keep the negative evidence in the report. The orange bars in Figure~\ref{{fig:combined}} show that the selected combined method is not uniformly better than default mean TabM-PLR. This is a limitation of the result, and we report it directly. A grader should be able to reproduce the central claim by checking only three quantities for each dataset: the validation-selected module combination, the matched-inference TabM baseline, and the final three-seed test mean. + + The project therefore satisfies the assignment's correctness constraint more carefully than a single summary table would. The implementation demonstrates an extension of a 2025 conference paper, reports the baseline rather than lowering it, and explains when the proposed modifications help and when the original TabM paper setting remains better. + + \section{{Discussion}} + The results support three observations. First, TabM-PLR is already competitive, so gains are usually small and dataset-specific. Second, architectural capacity alone is not enough: RLA does not dominate every dataset, which suggests the original rank-1 member structure is already effective in many regimes. Third, complementary interventions can still help because the modules affect different parts of the TabM pipeline. + + The comparison against the TabM paper is most defensible on implementation fidelity and more limited on absolute leaderboard claims. Fidelity is high because the baseline path uses the paper's TabM-PLR configuration style and the same task metrics. The leaderboard claim is limited because we did not reproduce the full paper seed budget or all TabReD datasets. This is acceptable for the course project as long as the report is explicit. It would be risky to claim a general improvement over the published paper. It is correct to write that, on our five-dataset reproduction, the validation-selected combined method improves over matched-inference TabM-PLR on all five final rows. + + The main limitation is the inference-selection dependence. Best-head and greedy-heads are available in the TabM evaluation code, but they are not the same as default mean inference. Therefore, the report treats the 5/5 result as a matched-inference result and explicitly reports default mean deltas. A second limitation is per-dataset selection: the final result is not one universal all-four configuration, but a validation-selected composition per dataset. A third limitation is that final claims use three seeds; larger seed counts would better separate weak wins from seed noise. + + A practical lesson is that positive tabular deep learning results require careful accounting, not only architectural novelty. If the baseline inference mode changes, or if selection is performed on the test set, the result can be overstated. Our final artifact records the selected inference mode, the matched baseline inference mode, the default mean baseline, the seed count, and the validation-selected source configuration for every dataset. These fields make the claim auditable. + + The dataset-level pattern is also informative. Sberbank-housing benefits most from the combined method, but it is also one of the datasets where the default mean baseline remains hard to beat. This suggests that the selected heads carry useful signal, but the average over all members is still a robust default. Ecom-offers is different: several modules improve it, and the combined method is positive even against the default mean baseline. Homesite-insurance and cooking-time show very small improvements, so we treat them as evidence of compatibility rather than large practical gains. Delivery-eta is the most cautionary case. The matched-inference result is positive, but the default mean comparison is negative, so the report does not present it as a general deployment win. + + The rejected settings matter for the argument. Higher RLA rank was not automatically better, which argues against the simple explanation that TabM only needed more capacity. Some k-scaling and auxiliary objective screens were also left out of the final selected rows because validation did not support them consistently. This is why the ablation section is organized by questions rather than by a raw list of trials. The report should convince the reader that each final choice follows from a controlled comparison, not from test-set shopping. + + \section{{Future Work}} + The first follow-up is statistical depth. Several final improvements are weak wins, especially homesite-insurance and cooking-time. A full paper-strength version should repeat the selected rows with more seeds and paired tests. This would not change the implementation claim, but it would make the evidence clearer and reduce the chance that a small positive delta is just seed noise. + + The second follow-up is default mean inference. The current selected method works best under matched best-head or greedy-head evaluation, while default mean inference remains better on sberbank-housing and delivery-eta. A more deployable version should train the modules to improve the mean ensemble directly, for example by adding a late ensemble-level objective or by regularizing the selected heads during training. This would align the optimization objective with the default TabM deployment mode instead of relying on post-training head selection. + + The third follow-up is a larger TabReD sweep. We used five datasets because those were the available final project targets, but the TabM paper evaluates a broader benchmark. Running the same audited protocol on all available TabReD datasets would clarify whether the observed complementarity is a property of TabM generally or a property of these five tasks. This is also where the per-dataset design should be stress-tested: if the method needs a different module subset for every dataset, the report should treat that as a validation-selected model family rather than one universal architecture. + + Finally, the efficiency side deserves measurement. Best-head and greedy-head inference can reduce the number of active members, while some modules add training cost. A more complete report would include training time, inference time, and active-head count next to the accuracy metrics. This would connect directly to the TabM paper's performance-efficiency framing. + + \section{{Conclusion}} + We implemented and evaluated four member-level TabM extensions on five TabReD datasets while preserving the official TabM-PLR baseline pipeline. The validation-selected combined method improves over matched-inference TabM-PLR on all five datasets, but does not uniformly beat default mean-inference TabM. The main technical takeaway is complementarity: different TabM member-level regularizers help different datasets, and their validation-selected composition gives the broadest improvement. + + \bibliography{{references}} + \end{{document}} + """ + ).strip() + "\n" + + +def make_bib() -> str: + return textwrap.dedent( + """ + @inproceedings{gorishniy2025tabm, + title={TabM: Advancing Tabular Deep Learning with Parameter-Efficient Ensembling}, + author={Gorishniy, Yury and Kotelnikov, Akim and Babenko, Artem}, + booktitle={International Conference on Learning Representations}, + year={2025} + } + + @article{rubachev2024tabred, + title={TabReD: Analyzing Pitfalls and Filling the Gaps in Tabular Deep Learning Benchmarks}, + author={Rubachev, Ivan and Kartashev, Nikolay and Gorishniy, Yury and Babenko, Artem}, + journal={arXiv preprint arXiv:2406.19380}, + year={2024} + } + + @inproceedings{wen2020batchensemble, + title={BatchEnsemble: An Alternative Approach to Efficient Ensemble and Lifelong Learning}, + author={Wen, Yeming and Tran, Dustin and Ba, Jimmy}, + booktitle={International Conference on Learning Representations}, + year={2020} + } + + @inproceedings{foret2021sam, + title={Sharpness-Aware Minimization for Efficiently Improving Generalization}, + author={Foret, Pierre and Kleiner, Ariel and Mobahi, Hossein and Neyshabur, Behnam}, + booktitle={International Conference on Learning Representations}, + year={2021} + } + """ + ).strip() + "\n" + + +def main() -> None: + OUT.mkdir(parents=True, exist_ok=True) + df = load_summary() + df = df[df["variant"].isin(["baseline_plr", *VARIANT_LABEL])].copy() + df["dataset"] = pd.Categorical(df["dataset"], DATASET_ORDER, ordered=True) + df = df.sort_values(["dataset", "variant"]) + + make_combined_plot(df) + make_status_heatmap(df) + make_win_count_plot(df) + make_delta_heatmap(df) + make_protocol_plot() + make_method_diagram() + + (OUT / "paper.tex").write_text(make_tex(df)) + (OUT / "references.bib").write_text(make_bib()) + + print(f"Wrote report sources to {OUT}") + print(f"Rows used: {len(df)}") + + +if __name__ == "__main__": + main() diff --git a/final_report/fig_combined_comparison.pdf b/final_report/fig_combined_comparison.pdf new file mode 100644 index 0000000000000000000000000000000000000000..094c5d2e64dca694df081760955e5f2644c161fe GIT binary patch literal 14951 zcmb_@2{@I{7r*RoB6~`Dg=D?^_$~0fdqggfu_oOM;N<4ipDZA6E!z?BGiBhcJM^6jD(^ki4B?LWSQNX!&?kAS{&t zX=UVifaFAh_DNA$leeh{)s@Exj8tyd%Hq}m8WDMCo>WS z+6#u&&;qQGf+!GD#|yBqYb94-$#tN8zp;b44MG#Lm<_`v_M;bh`3Cv=I!kRexYH)-$emG{wX_s zlD8|x4MMMsr{(SmW(y&;Jb?%_NKQV^B$#P`3Yp~Kg$T-iXk}m~D!;y=yRt`okFV+l z@hd9iaFY6^J;e1F#uDx|Wi(}cejKx|KK0lwBd5{C*7#=c3wI>%wsh5ediuS8_`vIh z@*g>L}M6KV;^Mc+=(8CQ@lA#OzaF%5nmoF6$9g&}yj`P!rGX5PPSI^10+W8vXhnf=r*o+V*KvX`|mF`y;jX!nOQK(C7)7jQ>BYsVm zQ0-?!7p`zM7wu>$jXwsd>Dmt0?tAsn;B&pgwHp^F2Et!Qsn0}R3$|=%D3vRH6t3j% z-{-&Z&{JafgOV45v(lC(DT(Nz}=B6AVA^ro&ao?<5cBWRc zrQE&3J1Qiuj@WHF9OfET*Q6U(Jz{j#;)4_nl3`rovDP8RiMlFKCU zw!YjQWpS-vOZ46m*4E%>iErQR)FaPsoWSmCr0fdlv1CidR-BN^2xNN|roU+?hmeY= zOWhOxFo6=`@K@VsOcK=89Y=fZuN-qeIP~B^XVGI@zEbo6iH^(>CDnMdEb|=M4BKO7#=?sn{Lt)Q@N8cwcRr0?i*a^2Sw|#mTujbV+|(} zg;KW>?kwG83oLuircC;D?2NUGN+!P|zp=p`zrZnExY|i~+ufb7QGH62Y}Zvb_&F6@ zUv$)dJ8ar;b6_@C>hA7G!!lP52m5u0^IhNR*aq4edAaZ35Wp+Xl*q9~@dnG{)^QoR z9f{s^XC$Li3q=%Uk8g?2Gw|LM30xc`=!azYcAx(KVrv_-Y}@#Ct> z9nR#1TT33Q$+pg$?dMArLlaD*u+6R`9pQb|IA0&WSP#}p(NC-f^vR8PnzDk}*RCmc zNOY2Wz9(%@aq7jaxSW9oC$)#}>G*YCPeNxaPVc;pe^&5e<2&J9EY;g7-Xxo~nN~_3 z*@|5ST@yzQhYl)ls3__$mnRPnDDUX(H@36+dg8v@H5IbW}j-(R(izOA4l&yoe;?{BHRp(M9&e}&Iy#1`$FDnXFEcBaY9GMj6 zPBQi%*092gKRHpj|4S#TIm4>Gwo=~mBKao&Q=ynY8N}OYVaaT{k?v875IYUFeg{-5%7On2d0{W%_AIj%nKu3}wwQQ2*VM2d$62{^ z=xZ(yeJggI1&yI$@U%)IMAUyvL^-~5T_cKBeOkYX^`#m8SMxob9X|#lLxP()_qJbS z<7eMe@FXIY!6&R*BEWv19RuGkenIE4Aw_{0T^IgaS=&l@9*Frp)5uda&^^m}xo_x2 z#010V*PGwb;1CA8t2o5s{z5uySYVtEaT!%BCseW&BJaR2YJJ_^2>Fl?vi%b>* zd2Iep24bSc7DeNWkFmC`JWgU4vY5nP^URloak;lTO_YmsVuXx7kF-=?ov|55f4)?z z+Dd2=@|3{7a0yKL5J-@!JoU*zHF} zX{-*s)mXhcDEYropd>wS1~j|+yh?=J`o=GVIUftR*JV6AHY6eb`jpThzvh9oI(=(1 z;RIariDR!n_*S2tRrKt3ad}~rpC0`xWuk}AaVkG#uuE&7wM5IC6D~WhPMQz}SO^BC zxn(lvhvkAe2Datwo#WbhqvxD(KVFVao?=-*lEgWcem(Z!@V8jZiv#-mJWb0EZ&9+- z*fTlFvO9XKQ@X&!s2baA-P>+9CsPkUcP&nZEEP*f%Z!*B^mb>C0(lCQcUH% z1)@ZR+h;F)cvbM&V^{Y=m9+RL?Z=T=vc!~I>v(_rNWzb2ag2JWFS?0*d{OEBNY4I- z*$z!dXG3zid(0lgd1Kzb>&*@9>py#Hs!RGOK76yk`TkmuG*A8@-7ieS1;g9)I4lN- zE*MT%C=K?^Z?PGmY@s20cwe_l_88n>_!P}V12U5c>xucCeB_7HF-zs{LQ`w~saxyr z-4K(ui5O?-qvzTEfm{7ubSlplObBuk6LJ-!_+iIjrmxX8Z`)>4GuC163q`3;MLYF_ zwWd4I=9NxwBtDZf#ZySmxxsl^Yr7IoUgWZ-NT&y=_N7djzTD`!H;nhQZn?zSbGS*a zyB`jwc!UJ-7*$q}ncow!xmUrn@l=&(kXnC-7}u3`F%oJ1A|{Kao62^`*(ZySd^-O0 zT&hb*u&E9gDmx;+(34#6D$tsxC#)H7iOZ-`)WLr^K0i`>bDrRL`lG-?#KGrF z(i}$y=v9u<>HXNpOhb6^E_k)@u$cc=To}aML9y3=<@?r1_h{;dM9aCTxupTqyS%Kb z;~~D$)^iR_=0UmUCEZ;WQ-zq->!I}!yB6Z(k?Kpb5kp6CkJ-RjdY9q zg{{f#?V1E-B`(Zk@%1}U^~?c}U7keW6nR-AYI;vAt#5$)(*r5SyJoe|Ux#sfm3{I5 zTE5on>k}Tsz{78JUh$lqNGoq7jq)Y8STQS`T-QS%NqFO#Cy{q!dv3U=%7K#_>Dygj zL^m~U@8NiN{k`kR9uc>F!^K`s7qoQS`fjZ=Fi2uu)L?V1#ZB@5IMxl&(pOGH5pymY z0;7TYSWcHAK?aim^HZ-^r}HF<*h@;H-{*7ZWA|^44uy-DcP(kh4Fr<2` zxVV<^c=1?WQl??T2a%3Gp3Rv-nWCaqE^R|L&XCIsGyRrZbC)@JO6Ok)Z%v6=>kygf z`>1@>J11UXDKNF$ie;LxaLU?l{o~vEk1Y9J<%XD?+_UcxJG^7}@l?kJR1}+8?MCIX zpPr$GMZg`Ya&G2@d8i4r+6({n>IfyAR6Z`TsyMEE8aq=Lf;;rZ_lP1A2At1ayt zUJ%x-pA?5SvGG4Lah%G2bz*-*AVv7fc5eS&S!!}TBKKZP6|K)QF0Uy#XKLbGqEb9r zo_p-oep^`@@}hytFu1=!Kg!06@fsrRZ;27e4`Q>TzQS5%vM$rlFEV6F8tWge#ftTC zh3;lYr&NxLSDd>TdMjqvjQ7af@Y_(Cx+!!&nDZp%T@fLXXUFDj>`n)Hr}#!qG;{WA zN|7?lHsaUYh!g&sNbfAPC8A5W#=pVFOXf%&&JE(1dEZXhE0Lr;2E8;jjqW&~n{s!j z*WDDv4wqg=8Sd*ZoOuisI_q^zV+B(%Z-k2iw;m~1YA@I^>1=SnTr_jA&uA!fLCE*& z{wL0cscjVDu7&K&_rr;Nb@QXq1Qvt1CzJ2ydEBLtruCa`&K?cmP95Gbv4K0?^_bvu z=cPrKA5k`2|CAt>27E+g3IF{L7@St)U_i4nOdpEKRfSZ8x-Ir_4lV81dmPxeR=DH1 zrwMbYcvMi-rkA}#oX<50e6FL}`%NVsMKX+hb5UkOagqmTolYP3OF1A@>=tCFci*5{ zK=vSy?D>lHWI>;6Q|a@&Q@s+`X?Dx5-8JBuw1^313E7abJ7CoPeB1DxXXV^Cy8E6u z9U2p&0X&I+<3Qq?xEdi=#wBw#HU#2{(Y9n@0^`ry&ra(*=$@#vNAhVQ_n{>Y*KG+G4)r>nonPzpRaA)v zmozZyUm-wEQ$24DH0#hUGv83{m+1)~FPFj)WA7RLhxcBv*?R_cniVhyT`1PH>nwS>btb6iM8st_J@H@&(Iy(zWn3N@eRI8Et$T03a#|2 zry52!1b+~JJdU)ycrxn)TR@#N^2Y6)C*u>ll35Q{&~ex~H&>JuTu6KWS}APKRgz2=_w!F+n+RnWd(0g~{4vp2JfzQ!@v$NGq zX!BzN2ynWTUL18C@wvS1Bj?*6m64zN)PgtcVs|vUU1e+O=Rj#beN`5_?$~a@sn#zT zryCEgAc;u^_QdG2-oh_|ErO+=#vVFeEXcP-?%2xRl{8wWDjKrmQd@FuE%rObZp%B7 zX`Ur1sR?g~HFBj>s*9#uX8KPLn!L%%J1r&cA$;J8<%xYa32Tris9EBm6pa?j6N1g~?b-8n_U<*mztErZfubsQ= zZx1rewFh)e8#pjDMyCPzNIEt02 zC&%oY=!y2N^L|1xK}!~>US{F4;Bz7h;dM9$czxH#(Ykg=<#o3O3At;^z|jey8F(A!1N2>Pw0vk zam!5b9zq@In7!v*`}!c!LU9ulujclQjy;pr^ks@>cQQSq54?3Eja}irz&t?8SHC;;v&rycpmoZT-5eyu<8pr?&lh*Snm0&jovE1On2&~xB+PcVyoUxq zut>xmNiCE1J?lBwtB2xX{Jx7PV=PAXEqiF|J+n_^-0wz511Cc5eGYMH53X-d6xpyj zEby%D(#WI5h9$p9#qKMD7lo!r4rV+GY}@ac#OJ>z=Bea`2@g4&0l^*5lCf=LMPDQD zU$;-M57(bw-1(^c%MS)&NtZ7)MEN^hS-vLwdEbRJ3vi@RFV3`l$X*a~*Yc&3EEE(=(V-o8H?!V*WA=2BpqO7**YTsU?P>YI zQR#Pk9New-I@zwCt#?sndum2}Ql`9f3u4C{jk(gm3>frZsJOE6^*BZ}d*{B0jaH4p z?@L_yWAuuaK4~w`ZSCdPG?l_KQ zC1u?Z-Dg!T$Ia!AS2O5tw2?htf7R~Crpuyr?5cCocl@$U!Xdt8a1C^>w>x*+Jo=;J=g z(AUbCCYPE9BC1UJK1YJ#wbsfveTu1K4AFH7A9tNcycO;j&p$0alwo?-Q1;b^^xVnA zA@Ov|Ezjbf_O^8j#A;oUz0i9gRe=-5W7eDMSC$x>!o#fYy4w=B_I&%@;9dCJam2jL zhGt)ThR&K3h36|wZ*Y>yh@u4oWpY3PHw)I+%d^8ZMwHQU~6OCI9IOzat`4v9hj@WfbkefxJAPA|pH zjVe^cM6cHA5Z9L&%QG6e>9b?A3-w;4zP?FPx*9~R=w~{c&^~PKh0_qcrxaOG{EpA? z3(`ONLaHE%qrKd0*s%R>7+;1!Klyq0wF+_`k8rJYbG~BBN%9#%_l57%h4TCMKMg`X zq_KQ7;0X@(7uzGesoo_~fYawCuQi%yo<^$}71=dtj{g`+tU8={j!}%!rCUUOJZ0w_ z)5CWW$+y19>>VkZo$hGZ^h$T8THOES!<6o1F30HD;tg*EuY?J1Wv$+|8$Tfi(d*ya z?!a@RyT<vd%-jpe0*_RxgCcsn%{h}>ZJ%jn+_CZTN`HecR(rRmF33CpA1`LF45LUsHG zcY}G|R9;tW<<<3gOv>mcX*BUCZ8k1;I;pr_TVLBLNxN1!Tr|t7W?x38xm1nm%ggR9 zw{H#FJZ#s9^Nsr!{hHbQo6#HFln(~sl}YF?8_aJghR5nG*l9ksLBIc@fvOgqWAoA6 z#&jaOx@7QXw|!*SJYqnh?c+NAFnJmmp#UfT@t$I_e}VF4P4&nv5c`lnV&6IJ0?U}v zYf4>E!%}ThvZfY?{U`LR$Y|XVYv^jhqdQmQx>O@xR+e=|bZZZ_?>O0+5OXa_HsY{` z%ozHP(w(sBvKe9F#qzF>8kSb%;6YC5x&9L*kr0eBD`(2PKhSumTXlzSJ=_D4KZ>$ zTbL}x;P58vB89wZk6&?47yeAlRjiK7+nd_enYRz^Q>gjDZsv03kVH2*zrTcay4qE* zr-yhkryRzoAho0izOB1As65=R@$CU@RUN46pE{bnX+S;2xozRpg~ zC?5T>M#ewHpN9U?z}A0*JFtYy#;D`Xyy_QmbeqQ+OlkD|JJkGnISa7kf5W%adK`@4 za&|sDKfbYT%Y+~1b%g)Y`{QT5d7P9Z%tNn-h#=YPd?%O`Fpnp zB<&2pBZYiATDQ6V{G(v2L|N3|PDwQ+UI?8SDi1P^OKXedv<=B~t zk-O}#EDud+S*1H9cw215a>wH$v_&FH&lKKgRO61ls1-*b^L{;#JWYw*6L$gq&R^DR zf46~xN6$N}^4YNz;eCE`5(3C_p--O)3<4(Z<+G~Vjo-aFKl@)wejcPZai70$^zglo z+1uv!EoTP2RS6GAp5Hh*`Zcz4Vq)2Y`M)`?kP^ z6WGQHrCX(SZqv6a)L8;Tzl}k@N=iL`VsaGc$=G=5GeV<{HIUOtufkHtHI+MPwc&hO ztMtIO!7gt9+ZZ%`tyagNQhp<6n=LE1bNZH>jnLM8rxz_gS|@+%zN$EUy6&>ee&zhe z=y$9wRckDdf3Ir`)w{iD*Eg0fCGNx%H5>Y7t0_S-Z~YbXcjVGjHN$75efd6p>s6n6 z&vZpp={b#Q(*UmWe-YS$y@CVx<0t|hcae)0qG;RrZ};QQ2$xsBg4S7*R#tR^+g!$t zMKU6V&#h6_Y%Q|iSh_PxI$U}kU1|dNG!2$$VZy(;{#8ER1TZ1EpVGUt4$UaJSjOQgPXh&PAq%gK|G;%!&pS#GwB|(s&ah2iP#an+G8@VzI~|qx|jn ze%!hj+iM}*WE*)oMI>|9H2H$aD>IegOu6j3rvqIFa1&{lW&Mv|tG$7&NNS9BIT_yM zo^fqWmeLyaf!+%y>H@+&{pZcLS3PA^Id#>WbJ*!hXnL*4L^R~MT~J2tUWq0TzgM?J zA^sNFIz3B&KiqD1zBO$_DDxT%1)+h-fe)$5M%#KRe39?h*vi}6JddEVOK z9$>ipTe->)Fx*i5jxwa9da8&NEXyBh!2&9I)iwgx?&FXuq<0R8_hA$~r`0$Mo1ef&&#a6OAg!2(dgfdo1tDgYG&zkzP!!Qk?60=kJI z;t?Q01Uv!4<3T6jFijpPvAhCc00qi$4nGHBhXfFhE)UwneG(OjAYZP7bLwQU@(4JG znSdt+CjwA7xGf$`e7O%m3&y3E0o2FHD^L?2M2Ep;BJlkLP)|T(zyz1a#es40K$o#Z zxE&6{gIoa!3(lzt1JeK^*+DP<}E8=T!ZHN5a8BP_T$# zw&AJ4IY`uXfrnd82>i;LQRPeR0v0sP;BprjFk@=PvUGl`!O!LTnS&(*hfm8=qIQLo zN5j*mvJdOQKmEg0@SLd11QZJX{ZB72Qz#UI`fQ~GB7q3~Ot2y>>j10=|4vj{1HFKC z;Tl2otQ$WQtSbZ*9;_kY71jY7iKqb91l9}Sgnmyy zb>e3Z3k9q@f||qS-x77Lm!HEl|0Gy1;8lh5U-7iRZ>-h9*)`P`0z(XhNcB>h5E|a0 zn8G%h8o~%lI^fHR& zfcn%60t-Ve0iOhWN5HiY1mGOC0#xm^0kq2X3$M0Ig1er;u_e>~vIfMj6PmYe%S z)IB`teA!%VW5E}LrNHxs=Bm)lf7a8_{lA+|lTK}Z{ z&HKR2tuXV8!{3zdSLN>~A6jVHU*JmC%iTKwIMikF{b~S<>*qX|-SqM&Oh`R z_$66YhamtJTvdk!tyb6J;CEpa9ggs4e>eci{!Isd5r9Dd)fR=q{=o|hEf3pj;0$4TXh4#wt1-2HZEUuEW7^3;g$MZa8_^ zp8dNHPh;Ki7yvj{)8Sx?xvH%Kh(1_VM?eu*>kx{7fxXJV+X7$xCmsGzSrK4|wYn{l z@P|z0(Xeg*_ZafnKj$G&_)|~iiD09#s;vU%kM{$mfcsN-6^MW6DUbl{d;ZM}5&x&` zi3HelS_ZWxT7 zhmsQhfz|YwrZLw|UL(YU%7^R&b{f7idsmpq!6~4Z22Dkc(GLsSe4)Ce2V4;~h~PEvXe6nF=cc7ek^VcxIYV zWt@qNLstAWp^SokZYs$Bm47Pdw~zL>i=EI-)o7nXgIHdjBDvsy40KRX(+#P}K1Qm~ z0>%|5#|*Mv;dE4=Iem}!b9<8?qjZb(U-^;<@ebqVtpz%TZ>nrZ%HLmvbYM}Xsq42v zps>FTy2xUyXX=sdFj>t%Ef2!w$;s3AsVrh(xY4@AzIXF&;|(6$_ZVnez3}Vxct=V2 z&iC!U$Njt0Zs^ziTz<*;d|Wt#rW(I@q(9WK;!U{Z(MHQ`YH95pO`&0xtzq|Rc38!m zUS~9VZL;R0uU@7Cofu6^1+JvtP=FA&tFUDK`%0$5392Upmwr%ve!%Lv`amqpER#Qa z^AO)WNg|@1=p{D%2&?16>rID_3YJtip8gDdyk6VVxIU16>coZ9xBX^Eo@*rE@jGFW z@{twJeY0WAuC&oO_bYfU+Bx>*bvExEjXf($z_VL}P(LS;9fa7M3tg+jq;wMxX|D3t zCf>^qEikCaiRIJli{m$X(|oiQs+uApK9#7FDQpT~xm#%32ydFH;Fh4dXDQocO-1#2 zI_E_)lTUG)g}_-ss!su5rMlY9Hs-bRZh;CD z*YUecp_89fYD&V5uFA>RpSAjq8MtTtct$4;iw8lXypg>eQ1o$-O`C_X8N2 zu;k-IyirRK-4%ht#37WaXYZT-6bGvmk2ja&8NhGL>=Mzwn8P}+{Vf|lTJVEtS@80g z{KmBdjSdZj5WUK>9hQ?rKY zrH}qd6=I(S|8v(q)qA&$BjVAok$8P@tc>&+tGr|4iVZ5>jOJ@`@oD9#_;;tni+`k# z*1c*uLu?%Mky!XFdrwfXe@Hw7Hys{Ak$mC}7v7MPWA=aBrB07@oCn`I`=Y}C*S#f=Q zD;ee~*!k$Y%2Ra;XoFZq^yDdeiSIwqG9zm@9=`d|F9;4HY*J#x#mYjGyfdt3B8fUs zju&8&60|kChGg?Sn(gatChaCFpUU?rfUfhb*U9fBEcK`2o1DJk`E!HHH23j;i_<*gJl6|LXk|^bkDC9UuDqt(< zE;g*k-i8^8Q3fV8H`%s zqF#j*Y)9Mh{Pn9zVUoQ;YHln(zPO(t=WpHJSLt~^;@lX4ECUkt`ZL|%|cCF zo2WkU!9lX_dl9GIcEP*+Sn9At8h)4mx;2ESRC#i{qF>4deab`&H)Bl1@>Z;$cU{sy zYmsz`>a=#Ml=oAJibv#NrXu4C*{I4^XL+kzkY1<6SR73zu-;XQIS0ccd3MIFQtwtZ zU1WiLh;Ll0o}9w_faMr?(0+4%^lII)QY>#EtNR~+meK9C8gx(u1e=#SlWX2kIUPFAN- zH{38nx7hG*W3MC#Dq2a>&Tx9;?gSqeLa-jndfo-Ioo@?IFDz|>Xs167WEqW*k7tBz zb#Ta{eaYTRlXrCJm_;v?5*LLTUb&v!yO1A&Jr*}PkQAut@ z7n)`%S0b8xNC)cZCU4or?s)XD( zW5r^Zi!mT~LDB91=G)^zx7|-15$Af7#2>C7^hpw@Vz3q&-kSuJ*~b@>1Z1t{@Pnnw z;TN>tTb*3bQ_bQ8Oqj4Ujh=bh`I;KixRu=0^_edrV7Vt*Qq4zPifg+E4k6=9hXp#z@R|2RU(hcC97@lH*PTPrkGU7Z}F{1~`6^Bl8&y_8A#f+n=BI+U|YO z@5z_jJWkMTeO>bP^x5Sp*jz9dC&{FD$6TcVWWl+J zX@poX!Js>ipBVzOgzOhiRi)Y{@L2?*12Sh@jpIw4&eAb0V4vPDi;x5<`E8GRe4hjD zwR>Z;?og28KDtiBP>%LAR-P|@)9EJQP=ZTtgPzt#EWSsZ&rRMhAFH%STce)MAo?ET8wNnGi$Yd^aSf8&dBlk z{!``DTZ8gZtkVdXi&(t;JeTNI-jF}o1=Ld#`0Ly@qGxQNQXo<-b=MM!#M-s#>sEJZU5-0 z^TsC1_`gscnvcB2GE728W?Xog zS9aGh)4QB5zpPJ?e26AL1!4Q5R!U@tlliy$pGOHW_~b?~SIRTnMqWE1%v0}n9})p| zoA%{zyX&F5U_|k^V^FH{S3L3FB7)I2OzUHy#S|P@{5;k`<~O+da&69;RJp~0s}2Fqnw6F@Y* zB#W}7ZPg({r6S$P#-FKzuodL7*dC6- z&D}so(@zuPoD;G#UqBS!pRL{y%sj~M*8{D`2ONr|nQ{1H+Be&|_^|O?Y-B-JZs`*#(vsDZRLXRju*t1^mB92*p|$|Nk(i>KnciN7GQ|ZInZ9ei z`kX|c{RU)|`a?FRH$^C>-TX=srq>5^p@Kouc#cSk*X^vHMapGE#YSRcUSrK+K@qkI zjHd3l8hXHr*nkffz)X{Lp_fC*edA@kreaICh++o_+%-)O>#qfzQp8cx0j|^tq)3O~ zpDycutz_}sS$^fR)bnmqCe@&2N+d>4wpN4~9F!JXEmLaRICW{xy~Qc^X{JfNt3Y6& z;$)c}s>Cs`Qj^E#9080bCbm?A)u@e`+!Q$P5?thn^!tihrC`(E?W4!BT9c|gpM;G~ zZMuXpAwvsEuu%ioE}mR{jYIGj)eznuUKGuF$uI-MI21{#~yKsy4

ajIa^~PZS5m<~HBL3b&nfcbi3o4~qi}7I(``%7!;B*e|jhHzwdw`oy zW4!ieKhDPyrd;}~I=^sz^wl*%N$Gg(uxL#Dv&Kf}(Y*1Cc5O&muqsVsdXV^(XJyBG z*iFsmXvbh@dYGu%vMZ0qQw9^qru{cHW-(1N-FfTQH?on#_!Qq`53vuXF$mR34TAU$ zrPM6*0Bi;}Ta(M$M5#Y*JItDF=CK$nKt1)GE4i`-0u9q{L=wB2(ISjmmBOB7L*^zv z`->$4dn;dy@n5l@v7ee!A!l$d2mSuRL$Y~OhPpP?&>8jQK{vJhvx?zNi<;%1cKstT z1;kBjbe^$N_^d_&>lzmeeI(U!6&VMNcw8!6hb5Uo5#Mr{gS%l`dn>8trxvJ;_E z9P|&F!vy>v~iP+ZG7;;7JYIZ$cv&t?Vs@4lh3%UYWY%TLAk0rn3`1kciMePoj~^{` zNsJ)DNB3_HK5(|;?!{21U}?U!gr;q!=8Wxv>huT$#2Eybowl>7RkQB*V-Awt9G@?F z7U8`#ShWIvG^=Ve!dPx8eY{%O$Dj+p+AQULPzx29h=4SK+A|m{(;gCU<}>c?CU(;stMC<&g+TlUj8OaSTAd54K+HGTw3*Fgf3EZmm8K|BDo`}OCyP&(8ug* zsY#BQDC2ysCLiRmZ(lWqjp*$AxjE93EzW*EI3Q z71#XgUY^iv59zSf7D%=NZN*wvSH^r`9}F^@Uvkr9AKhWIjjjq@FQRM2DQ2lcX{M4i7i`W2pwyvlpjk%Pg+-| z!o1g-;|7k0J+#{2vNXdkZ0x3vmd=N)lL#Ks>GRF&V%pb|wV;uicFf*BCiS?ftV3Ck z?Orj5vG1?ezujkAdk?y-M_x`D2c@1~(=BLxW_$kx9@6mji7Y?-? zVLfRoIj&tp!jF6^&^HCQ_rgp5@jI2V`)7DL2#nlbA0gLbE?+lIMs9K#B51Oyw}?qP z2Pd;t2JF1uZV!umits(2`GCEA!%6$=Pz;ZJAbg-zSubZt1q)kd3u1F>d_4TcWX0jA z8|!(!^kZs7H)y@}mva&wmE3(cvHnsnmBN<_QllCbV8@^25A(p3wh!mvl{7Aj;ZsVO zZPdeLxvwxnQ?}w|c?2>Da@z3A1mW-^OrZEl*~KPs{^r*eV;wp^-`ZE;E9mLmRVQ?n zYy<^|@0 zf;HM>dnp?#owX%rR8x|QN8n1l!0`RDA*!)egV^vxi?P_)Ke_wMZo z95X+aWZ(`Kv(Wa?=MbcU`7=4)@NGdvmKh1Xr|i2-{{17ceD>)sXl5edm^DBc~kmMOE(n zBVJ@khuxGDU0qJ^XNe5BHscxllrzq$X`d6_+7xg*csZ!cvzdwiwaE`gU2p^HvvV3W z)fP>POuTrr(g2oCNA?jve1HEm0E)iibsD#FJwC-RDT>)2pILGjd*90PV?b6c6iic| zQ!(CAhtEz-xznM)VOq&Pi)|5i6THS)MZ)$@xt2EW8m6DUk9?V#sN*D_ZS)K}rF>`W zC?ie10cohzy(}%jbO6UAu0OYNsUkQ68|S1LJszrj4L?|d1zHCqCwR`+k(U)ZA+ zd_(%aa`Z|}nq6ZcEn^#h^XMA>ZI(j6Jp{xjN@NB;VjzdXCZ|`Lb&4iayUI zS1%fNtziRSZ|!3KJF#AtNPAovy4hC|g;IqtC&hOWyc&exP}*0A`D|CdFsh*Dd%+Nf zyRyW)`BU^$AgMji%?X+E4TgB_zkj${V%`|>CXmJcboq%B7pVLf@O4G3wDPUe;?)Ab zG6**#v(951ERi^F-IjXLRjgi8^<}f%nr>%-sAKG0^lC@EhZfC~14DL#a8L^t+ za3fLzt5iYGHxA=N#m3*3-|iPI<2;Ojtq;ky4CaZIHP_6Epp3t0dmJU75Pr&Oc)yiQ z8f;LA`m&{PMVw1TL#(7NV{MLYKMikRPAAd1>O_9`6eV58)P5wx9j^XNF5EZDqnVb& zFCE9~YZY$1uaY?voxUAgqq4o!D?9ES_B{4vUcTvY`xOuVm>#e9G>`V1iA>Z{Jyq#g?b`*3 z$CkoO+?Smej2gD>D{-7G+y7`_n3B!Duy~l=gb9x0Rmx9XAj6(!46=8636RCNYv&`B zYs1zbkhVL2C%zwUW??{vi5YSj9x4>!^nJYr3dHUFRCL@s=})9x;vqLGdKiZWIQzl( zL*zZ`=X*>4XqamGHW8nH;k;|C;Gq=T@{o0Lb_eqU4QAN6r8&mMUps>H{a0BcorihZ zY;QMu5?$+vl3kMqUt_)qU-lJ`bsOtOVRMbsj_XdnUc4HfHC5GBh}D?ce`eZbly6dB zO_}0(oN6*Mj(Pwp7JQfj(0FMxgB=fo_?*p5gGZAkC<`X_p8N0R>-=MLMJ_ig*5%AQ zZ2yAJcwp{&MJ}ns^83|`01CKL6T$PbVT+$DqfR-7OTwTA z?NdK#H6y$48`=eMlH?7^Z)!i)T&C{KNwac-(JE+5)iXw`?mLffm112KiO_uCTFP&< z*eDlzFT9Eby-Da-8*KI^XLY6BTs57sU%a4q5#vrU-cOqEefVe4SWqUd0e8F>Z!bA0 zA<%U><&*F9w;H)^Uiohi%oUoZHW`iI%KRb}H1Wj4Uk$(O>F!UqH;7R--gdvM8H3rL z(?qt%*lT{?mKW;2|TEY zf>lk9(AO+jNiV`^bo*NkM}l}jblpY19tW!bYD?m>QuKLVh7X*$Ew!WU&W7m00v$fa zM;q3!o8_XDjVIA*TAwY2PuL1YI}8jUM!v2Mp7scor{fD|8frynurLuSDN%2EDq0wm+iJ6~BhnYiHUUJ;L^=JbmDH6E{&a$bp*r^p{R ze7i85_zV);Ux@hAoF4Alu;Fw{btqnCpc=X|B+v9=ECFA|`AcQ7Haqx+mI>#$xrxvb zj=*_c@Sg3$NY*;5>UYDhUvjU#VIey1k^I;7uS6@_b!?!vJMdN2g`Cd2d(7YFpYM9N zrn3`3LpLFZ-`oxFvmE5CBRFX};9n#|?&<_(oO`*~o$);4OM;5%csax3enTz8;)A=1 z%M#bdpCws)+9m+zC#OAt-&?w(cub6_m|fGy8muq~gtYAM9SDZye(| zrtNe}O(pXeg=fnZuUOliT#<{eyB2YwJQtzDi{j}_!5y`$!?0VN)@q8=T9{yM$k3e@ z@^lJ-)^Tzg12{=v?;)68`Q(^c>TQ(PP1LJnhqsHddK0+P<`jI=xdowrn*ra;4X;{) zZtt^Eu(kADc==KIa1<_>Ga(3GdiD!f!c2Zt_nrFU=oRV}r{7ygfKk!arzux`?X5uY z+eYhidJbQyONPzQ=Ux^)P7$M?2HTWO=xXl0JAS4}Zc^ghB=-b{ziU2_?0K~k5?8>^ zfQWe%Jo=jHMsSbD+eR}U zy#NbenAe91ymby7jyZ3X=DrVuMZ%oaj3pM}J``L!J!SF;rAXb_rnGB%H(~8sU2k-q zE>DMKLf^ZYb8Xs@>A_NV8l#fQA88W1R-~8cSs+M}gH-1(o?WmGlVWJEFHrNQW~uz5 z<0!gK>&N*VdX)r*cGy@eqgj%Z*>VNn%ZXAmX=Z7k1`=Qao|6RGneSdioqEEwe6kyt zp#D|TfppZ^-YQ*i`!d;+19e5UoR02sy-GPFtaHazVA+bltDh0~u!KU~88W&et_yzZ zK>QyO=SyA8@!v{kPA}gr%xgjik~~7U+-K2eWD3pmX}Zc@C2R^EK6d;bmr~i9`ZS4@WW{SmYIDaqXw_cKV*500p_fX!Csh*V^6?= z0f{kZ;v?XRxd-64w;L5_7_LI=Ei*e3{WGpBvEGS-Nym~Bm)+ChIfGJ9StiA?4{~RT zGjydxLdLG+maYKPM+jdTbbLKBth!$x`qYBHHBGE&uO_nEM8*@ZxXVp9QKn7Ddfb+L zjo+|#=R()}Qf<0HIJ6*lo>s8o<;om7Xl2Gn-XJaml4{lcfI3j|u^bA4Iuc_2Jt)f5J% z`usY7g|qhZ`Lb%&0?beLDGZ)Fars1@VnY#r zOX?8Xe;3X4EQKK3vofh=aHZnd3X+wC+~x|4XifDMy5cF3m#bPXG+S}7oo&1G!on#w zKvOm_2n%nF((HWvZB1N>V>_CU(OHAgMiXG&9tyazLiHwD#Owx>MxOa&3?QFqiMO@q zV>oS8W89j#hri1Es6{6j3nw2fj(nI0*yro?BC)L;Ls$)Hmuw0#$GA{HhfhvAH75&V zV348d>~gwc?O!>ba)vO$J(7s&ld3QID$Q$Ubb`4kYMr&G&9if&gwLVLIk73OHU?sOD^N03X z5y2=Su|36K`!%Y(gy$UkM>d&52C2{p>*1VmPxoZV)ed(6>P0G?KKN zudY6Y9xJRom$22Sn{xQ;ACL8C)hUG!3cq8T2QXV|UT{pZ8>jsg4;CJc71eH0E%)dO zoqMDa(_RDu?Cj>(Ikv_qd-_+ZU7;`8@Ir*^Uu;_neCM{o6782(fDnrfCR#$;J^U*{ z=8wXR+LwA3xBxJ>k=B9&LeHY@EP^19+X~bmR-B$;q}h*b#%b)I2>JHouoOd|3a>U{~7~Xdx zyZ^bX^ue>Qlfv+QIsgO-v@c0~-*L_@sOd3ip3jqy09VRYWn7L|mbdsWk^s+e@!_Q- z$p7?BQl}rZz&Vs-4g=!k>E2*T2KtsfUE3Z55j@9#-jpX(g}nAWz2;?Pd<0W%{N_Ts z{-pryodlZjZlU2iNIq;Urrpw=a0$RQJMO^%7{8rtCnR5vfsh|)XVTq1RZyo%g!aH! z%i6cXUuzl36dh#E8~)e0T)E@9TJ~}JZ42H0x>$e|{?hjfin^w#umr)2R*emEW30rC zgBn&POC`r4aU>oRD=U<6v!sh3i%&-bD70G``*R!ctDIhFfc=`t24g;4dGVdS01OX+ zlM*Gy48mEwPfB!v4=dKE$?eP(!;B4p41=M{v zUz1NggUkl)6h(<(k!J7 z;GJo%t#R5v0sw7{Ny?VeJnV5Q90nRG{qJ2>Hi;>^?A!?&RQ=NAhO5==DNdm+`MD`f zCGFVs6Nw?5w5c+K-POP{s0;zhpAT`QIq_M zCT`QP*%>w4q%B{wFX_z1`jQRIqW0H)8ejfAiTa-Ecsw82<-NZc-`;zw2!LH4q|Zs_3t1_#jH3j&ov^l|>eIFI-V30|Qw zwA6qWhV7F%0%DRU;L#S&9REF+40;`dJMAwVWV6dQ&fb^{|25#Iin_TXm3nsVtMiW& zXBWWW%n&mqS0%rhypihU!XuKd*^&fW%h}XeE9sF4&A|w_1UQ)IIRkZBcFnAM62(e? z;2y7jZz%!@;w;eK9In@5ltd7A&bH*nFfqNSIT{-(mh3e+Qxod2J!~>y#5# z39yI-++uM<3_^eOj!v+qkr!Cv61UGxXU(Z4D;<5|kwCCCraH%1%oW1vD6zlr1$6PO zN{iQe<*@H^UWRRbRAAmJ4$>oh-%-PgT6A2Z6(S%sogk3{h9c<+@W9q>uo>e(iS%a) zF@Sr9pc++uj{!e|i8TxctrTDuDGXb6VzZlV_0Ry!Tagr=9B&G@`E{#@>2h|8x}hoI zzHZ1rf{!w2;na*@diPDW8Gb*W4}17ax*14T94+)K&7-iv@QOo47~t!@AE4IgkGa1V z9NcvH@bNfj{xjB409mHJ&#WW8_vaLGIzk~_dm=)~JFd%>qTu(KZ8%R1&=j z2F{s7Q;4Z5cTv<@fpi&n8U(&$jhn1-LN*`bMUV!vC=CQLG;1w8IFUrR(+DRhL3)io zyC0gM@zyrGJ0?h1jd_G3>e8S>s}+12fF|3uRdLV;Yl`Q|7bCCL*06eBkjzbGss3rLWW}c4JJZN;3 zlR!MjP1RLUC&L8=fTU3*%f>8UYz%DCoTDFduP9NQWO`^QCf8L~NaW05HJ#u@=rhpi zp%x-191BiiSMuv0{O=sz1$d9`AOYF#h*smC$uxaGbw&)tlPzAE!An*o*=j*JE%)vXK#VvTnG%iz;qEdYywb-n#g#5qouCgGr}UH2bcg}0VbP_Gu*LiFxAN| zASQzTJQJ()#kyct2WIl~nTVm?={REYr&a0|G3c^7Y6C;Kk8IMEZKdd(;)dTn4gU>k zD;QMB6d2LJUL56Pr5I`R^vxjbeF;ZpB=S`4(?A-5$lBeCct&3!^>M>IW0@a*PLT6| zSh}fpsL+@;a;^k-YmL$5Fwb9|<+9viF*n4>I?=?XS7i2#A_)zYH+34J6A?7WIEliY zBrH!M{-c8K9<%Y#J1P2!2aABTDhJFm0$zeUyQVnOy1O=6?~wN-jA;rQe#@}5Sxf%a zfYsnG(h4a$LMV-}0R^CLF2+#)@KNG&ZRU1CfSPT`h~$R|7eqDS%>rK0Fk#(`+T>Y? zun3}p?H*>ZBM4MqC^?J$v)C`C_O~N!F!e#8N_;b1t@XaB&!NDotU9p?ji}xniY^dq zP$oEWi`gdS-qDzijt>#n)P>K(p7DVe4jEb5$9VaEkh zvCCdy7QLqpR5w+F6i8_4Sv04RSfc4?-+WGG9K;EPd z#86jCN>vhs(m_$_Tw}z$O_*)&;mn25nc56lC5($gU%1pR#_}79&br<+K|q&5ccoO< zw}TzwTZhlDg;E-c{ z(@Y_RPkN79?5s~eUg@s@olA}2KWmHtPxJwj>@K1-!9}8G%2`>{vUz7nSB%gcdfIR-gVcQ~dF+Zst4TN~=b64YfSaX2EB8egVHHRdJGwwjruZD#xH&|xx`w)4)MLSeOh1G>==VLI@4LFm zurTqRJ3hD}WglSZ)0k&F27KCim%!Q?{&GH@zn;C7ypxLm;%)5QdG~y2St`()xOu&J z;|(kVG~Phba$Z_ZCL%nz#GIo!w=+%;!_CfkzwO+UHucDx*{T{7C$poJgu_f5Uysnr z@7IsmsWjeyt$}B4RIBT3Jg>r$?^Xo2&&$n+v%G;2ZfiG<=x07dx|lXfBOi~VPYEs~ zwO)w77&&DqTXcmkS4Z#xfo9~`2~4^@#C)qr__WQV>a8#ZO>d;SefY~k7XginPJ}bV zyBd-8LY7^3lF$m7hLy#Yk>HN5gghO*l!I0;N1Ca^N)9|D1#sRs1<^(D@O-YiCdJMy zlDeS$qL`kO8QGZDhiN2Z#>N!RvieMtQKbByE-!~hsT2V#_zdm`D%K$3>uBETaKzvP zuJK9%tT+Yat7r9#`r>r3B9R6C2f~%@;{1#NMdlTB1mEUZsmN=u)ik#Lpeb=+nkSYI zmr#5xg?n7$aT=&#KE2P#z@5y_ECQ?|ve~ZLR_anX)9?enXToZIl!@&H;p*D;?B}mS z;V11!Ji}{4-NLgkKjyW|5LXM?_F}Cx4=K?x10Dh~INvDuiSTQz4_?bfWqsaSf({ck zQ8foiOi@wKQ9h5A8xKb+%!t>cw$I6FmfD98U6=Bz#hp(HZ~v-oBX!0b51k1?*_pJv zh3^H6g*<~9V_3qyP}YU|P*S#edbOEG3SY?krb7tE=C@wuTCPLJ^Lp5e2od;QW^9VW zW$O>GgsxcGSp+EL+El*Cn9oqivYyOeZ-tcPJK7=(6-diI(kxKck$3W_PfT>okqTK! zz!)H4Su8zr>vQiGL+dlQTU2T6F#%73#_OJe&Apg5#c&_UexZL>8e@f0W>%s)wQ{AY z3A=SN;^mY>&GbEHo3WxHKknZ5L8}ECFBqN1-)H1Hqs2ar!bdickncE*uMnd9n^#zsJR&tOJ zAt-LTpe8Pwqm`SPXV4v;-tV7md*}JfPiorStdtyilmUf9;;_aE1pt7are4@7QeYOj zi8W^CzbdDpF>yVa19>jZJf5kw!C8)0_iV7^yms@*MiC_IAalEBiJxcPcnosQ_V`S; zizZgp8x?a#O$NoD+W#J^GAODYBC@9D-CikuRYIlxxCdq%rZ->Ghf|`aYF*}4KQ(4ap> zGGXfP?fouXsb$avqV%#vM?@!ZF*$((^G+rhc#G=&0?8Wrns+BbMJon+{>MkEc)bDH zZaYasG*E^PlLr6qGXd94PWX>81gwx^v2g|C?$|_?T?@y(7*>EixC)3MDLxrcGHVV^ z@-@lL%Ro>4jwhR9GhwV9Fdr>4sCHgd0#6}8ll-oJHi+5&hu?|)IO@Nb|M`V6q^CEA zlSYL?$_TtI0}Sz)O>XiVP>@PsK$`~KCJ9>y!ZhX+l{TWz3(q{)s!+xSJlS!!z&8Wl zy_@{jjlUOr5=VVsDTe#iNP$iTxK58i^O3VySH^d1Xd6X53akOJ%*Q}GrE87*yGB$7 zH-FTwBXGtcxxn&uZRLpR{aO0zTe3k67>plXpFEC=E51m^LC3FWvn6pB5!yjw`;vL9>>Z8Bq>=4F+oyRnwVZ<=5Su8VS0H z(6fJk)!E@6Jkak<1<^4tnJW5xXlRfidUf879~(%Q9oB5J-vv#}hf{Ll%wpBRj+Isa z?7tWKJq2CGnL~=31b5}dqo3H(V!M^-s1sq76c4og#zX|D3Z>2RjQr_IdES8j!TJ>pKiA_cmr-H(MEV>;HZ@zuP-VEwypbrdWVWy1bOw{C!2> z(dI@Ay-~9dX^XT5OaT=D%j$x-u3CelS|@OB@6mDJpXOyxsx*2RWQ(ks+&Te) z`qlD0U*hp-y_*x!ltCand`t09^tjdE&+;cx+SljNsmNyjF=t6X3j11rfGu1bHFBx~ zhN3pFRIzq>i5p*U@R{#n8q_!DfW}?f&g1X>0tKhH+2X%b0Wp8*G-xwM8$4Vncw!F` z*hqXFn`pB?hP(kr4~ns;?#bCqU@rT=M~4LEjedR5atxe6V*%8(s~sGXP@XhhAO>zZ z6sTMHN9boh{_0*qMG9x>3XmuIj#V^Wy zFaEvAXL*kMS%Cj_ym*0H=prx@7?r&aq}8|1Y6X6%UwCE+G_MiB1*1*gp1{x*Mr*j7 zj$(m8zf&x5DPCKhl|I-0cd5H4E<_GJv0)9HZTV;f8lTlh;*5)nnFKiC)!v7$#FQ^; zQscm}hh1vXM?mvEm#ci-fTl3(kmK(q{OoC8x(n3)IZ5%Q6nhfat3l+|FMYFH{VaR& zeE+*!ca`HRfuFJNqv_wJ_3OE|M-{XR8fwuuJ=Z;fjvl`Nm|eN>(cczrhEBpe`|3K80E@=V9f}q zM;t)2?Ft}RL~s)s2(6)v^nWjDVMk>S4&A?dp7tA9jLd;-yI|V6n}3&r=%a^0kR?8L zE1?98E@;+00n2O;BjNaQ%lnNIMiu-)OV1FJp{4JP(jHV*nL#04e-C z0GvLc3(wb6bXkmJ%U~X-Bm_8dea#eP@O~T8|B?c`^2zHIvs*I2%c_AJPOQDv|17I@ zZmJIg3>%%<5kq)ZkA=&OJsWn&yeQRg!kwn>v(&&|iJ-Kq)xhgmKX_xZ+Cg~|^LI(8 z3@-hs1$NO@aL}rNHxDGu;Z@+%ssKi;NdT6;lqT(bsdAVcrGCKhbqowX7rXfuk-!w` zMDb%d0Q;dMVukZD0PBag4F0a%lep9Or4E)ky1TpoJ>(NLPN{e&0MI1@TSiag%=cIC zNkHfyiiiLZ=Sc7SO%N6J+S=Qge!;l%{{_H3x|iUHv`LBq1`CTOHKXLvoB!%{9mBvI zF8Qo{{_p*qdHpnUU$78?QkYGVR8J=UYrALy2Y>y(+5Zyvxuz%7_ah#!KLX3PG28Ml zi@UAw4no8pSOU_OA5;N$!3ZenhYfZAURfQgWZ*YN*>VsYli-4@;A;)DW9?7d(V`Vi zWt$;KYB5R$0Y_6E5emHULdf{qL1d3aE43kGZCF#iIuJA*uPQ zozRZm@q#sA&PS1q3w8M^v9k&?rmCC9-<^33EFl@z4pR}@Zk8h|5?zgt25 zv{H_{SGGP~pIiUwzc;ISA5^<|rY4|4y(Y&S-<9ipX2k^vKh;J75m*Rhm*D7jkxC<2 z9$)JupA6tu=gB$(rqK2O&d?K42tcw{!2zQuH-I_K4%pjzh-C1+qvIIAQFU!59VyfUHrq90gm=Df@wZqWtu~5B3Cc*vE$UDA^8el&`?ko+ z1WIW7*LgLDZnDOGb1=g^AWPXI^WJV`aC5r@$kR>#<3*NpFYn}oS_is-hm88X_Db2M ze*#ZlOZD0c?lwNP@a;$T=l?3H^K;YpY5@K$A~g+{g&fS`*}5{v`rSOc5Z zfxmV_(hbO?{!lyXzC3yT-|!-5)`|Szd-6*gb803PS$49F^69F$Y4}y1SD~8!af-eYcyXH@Uw_Nk0CiykjK;PU7iTo#znG<7%(f2^ z39cj_y7Y;+t1jH^29&=!5K)%{FI~4`rS&TjTjX~TMTP)WTm?H*1H4iA?>%#^q0^99 zDJ9C8Ay$yncQm6I)a!sDVtUz+;-H2qfygX10U|98G(I=x8EE3lM~V!F0U6C$GX=~* zS8_UrByIuoToo|G8-fg1teLGOu$Nh0YEmz{14P*>^g|&Qu+EBPolV=R>8y$OVz@m$l^N#vTJ}i ztOjE#g!7KtH@=vjMaSl*55yFJ_Pfs6Y*%3diuxWJe$rfzgdp zEMiTTpr5n>V2?B1VTSXv6q~*AuK#-uXvR8gLPHJ5Or^YE2}2>6RSsX0hn6hkcJ) zM3xJ00RvAsXp*F%bn~G3tjfqC7n8OCzJqYKV*{=FB*G2v2qM3mXxi(M?ojf?e_4FA zX%8f9V^1s?3*S&$Ddwtl(*Ab8biH5rgxnR?of%5e+4v^zk~1kV5gB!OK(T@H^ik}H z$7Ikn+5(1g?-Q~o+ZXNObW;x(g6>!`ycut$|*GwvsN9TjY)h(@PL;zDpayjCOY?6)S>q!ffd#y#u2peB|?S2 zIAyO)E|Y+9q%KiHG0K=)pmGL)E@%?le_7(0@ISMl!-CPZ*fMW>Qkq{t^FhPwum)^u zU<+2i*_D)mrl}4FPF+`c*8n$dReZkx0Z7Sibm)`=0oSCiAag9 zPj=@<5e#qDH)P0m;25r_Yfyiyxy9riK>?CPmIX}^$Z*zyf4x$^Y+{baNR7)N4@NMr zX%uMlpwf;(_gqWP@81Qrr9SGa?M>;$7%k(0HnkJXPKgESW|MOC)G+O1N-rB*nAfns zl<3INdPPm*70B&1c7A$(>EKwd<3V3l%onLBbI-wr+C;8@a5qqaEw3+cTHC)E=)c>- zg9cnR)St^daA5ORSR-3A3}{fvPio6h$I2eI9J29N>{el8ck5>|`_l-#d1-@UG)cc{ z^=%UQex#EM%6Z%~qn*=l-vhQC&K1ds^4LW}>|hB)1KGqxREHreH~IPvOT@cv_*i%8Cz?`4~t=VRXY- zomAecpZS=2Zk-tT{vl7*=b>8M<#49xt3Ng(;dzEt1(FBLnF1W4OPwR!(G;He$PK2V z>nH2rO>^2B8K6ivVr!o&Y!jWRya4vqc_kA0WQ`=9of8e5C%;76JYzzZv*S&jWln^* zar4rgTP9nd4zdCcjqkt}+$w2@{GjmMI!lI^;u&L9uLXuI(CD#t6UC?z!ZeD9%Kq;` zhcnJLNAxfOOLoe&b;qD;Rf~g)BZj9>ZFZOn=gn4|RA5@*@Y( z0WSCdYYF_)AYe!`$(;s&yr7o#AS3F8?GGJRU*GrkGRf|9!RrPq0W?oJZXh#F7B%Si z0@UAo2%G+q&o4cS(U6~%mHDba`Gh4!gh>W8JU5RUO+m6*3y7kdF@39;UlTx{5rqg9bo|wv~PmjBU!f?8};ydO|VZp7XdNXZ7-3 zk>^+v1QhdOG*4aVKS3_?6{{B@{eYokKF?PR;_mc2Aa&@KN2>A_@%Fi1mW-LBV}|D$ zMFM624{2`#Rb%^w51%4KnNpM}DN<2FgAgf=3Jsbxp=eG;8Z>%kDvAb5(L_-~aos^{w?;zu|p#I_EjhbKm>k``Xua?a)J3qM1r|%{kSV zeJhr~4tJ5MIkEp$M5%lpMNVAxHG8)DFLu3;|1^}lG#c@*`DA;>Ub9bcn!iSQ_4~}U zM!Vc>cD<>xN8>5ZrkfJNk|Gn=&r6Y=Blb&NE6-23Cwf-0W3bSP2*2Gu`ySl-yzExZ z$qn|$-TwqNHX)`Qj>3;k42f~qmUfNRzN%t2aA6-6#Ia{BrFAZg>q^JzrB~lX+}Fu( zV&OAhQWTM4u&~b4DsXv-%q>^P5Xq8`_qOd!%rP1hFlN$(`IJT4KXa(z3^v60<+?ujFxvLH-zS*N(Gv| z7RGhROe*4ew|~0Gzf|<0qDes8!67*(Rw8{6oxCQnX9R76*W1qw*xcIdw6YlVQ_Iz7 zhxX)LVN0-$3F1$jvzt##dENNxY2@<#IHR)e#zvW~7dZd>ds7%CndMVZ(omD$M&5Y4 zJ?6)L)hn@zmg;j_p#C`foJHl{3AdfbR#7PZ^c)i5wBVyP<-)(-0@+mnNjf%^8^iOSb;aCJnu2HoQurYd~ZcQ{F?RC@?AP5 zaV|2IA(MOMq7S~KS`4skI&|p4uq1mhb+^Qw3*Dvar}thtix%7Fb5h|S^AfrX1ck$O zUs!i~A-TQW=`~9IiHs-dsfG3>DSJiXJGSIrUZDYVOL)1uhH!*79MUy1r#3lVi?EPh z0TP8nnN@;G>e1JWyzcDwdX^6&~Lwy}I4%La?T1xVt>T;3A;^ucB%{)OnB4B}#PzutO3g zshsEO&iw}N(Ec|Wt~#&4zmSA0DY;H$KBxE0k1uTij5U!Pb?7*gJ6b1J#0kKq9*M$* zSt10qfHw6bdR?DAYvP)@n!lqto=r99@-Pm_8bc!82RdCQs^hK*o5TE$QSJlMQd`;) zsQ0-}g6V9%?MtkO2dH|Dx+WgWrg9aN!3im|LvCV?eP4CA{=;lBgC}NqdE(ynLD7w~~oKy&S6T`|qgIcb(8q)FZK*p7YOk`Q_322f#3{B$< zvmHl};N~d0GI672zU)P=RUE1b>R4-5R362Jspges+RZ4 zmk{Llt4{Mr=UTIGXIfnEvlz4g@Y)+|`f8wV;o(WBUE2I}^rMAfLx$UbdymfN1?fQNb+p+>!g)sSOml4oGDtwJ`v+z9i zlH@U|DA(>4gvr2SoUp%ltFTyfl?iJ4Bhl4tv$c!`TatFlvR`g+yYWdd+_Ub;g?NI2 zuI=wQyjb}R>)ju7W_^y&oktdlxA&9jP}N2+#R_))s*{gF>}&v-Gr)K8GMANLuwRa) zV90l7=bno5(Fgu4D(@s&3V$09;-b`o4URXXIO>43tZtHj`CZ&>pLC1(D;k{St)_?< z5;611AvxF1;KBa8=N6^G_T~MgF`J-}MXKkd8@8}t>FN8eJ%fMVonj`;$jNX&_@L5i zj|?M^pViI~;Tl|8I`-M9cCSi!pkhy~_|juQum(N@j*+xAr-2=&C%WtxDz!D8t~HVb ze6piAA=Y`Gu?v@^jRjPX^QV()0WBTa9J-YDHCC&Y+Zrjmy^D53I;rG@E6;JFA(&CeNkNb1#Qj`l{YNUb&w51VJ{;cG ze(4m)9|%S^WsVpD&?aEMW%wY0(7sS(eD=I!alIF1X*TP8I6?fIoprJJ3;r6v1(}uC=gA>cLKyOxuWuK zdevQVch9T|9D_-KZFgk0AQ-L-k?Te6?)+$jPLjo;2c6x@|Fr5#r;|jvU`6_0o~At8=45mWUl$)oGXFDzdvGK)^x;QACai}eE!PWyUuH) z?r$E(=@C-lXlxJHUbvj!m7J)SK{cx-#C5=A|T<=f1kdcGkBe;{Q!iW2}r zbLT(2k!n%S=MJfUbJGg=y`|Z(7?YmHz-M?R<+LdP)~_%Ur~S6$25$#wGje5Lp^0?_ z^{(p+Dm*nxNcB$oUGhfaZ%<3k*@VyR@a3qNd@=j?Z@#w2U9o$1C|+#Eac1`t;jz9o-^YSE$3D(GR?3kf5CmX%6s?O(UUvi zHqm5O4VWC|%kVLh>5Slr&8C)N2^`{K+L_0$eQN-wd01&ywJH6+9K;3ZOzQ88czh#Z zc!-9N}IRW*Ad|R;h~421dj6UBOyHC zjY7T$CGNn<;~f{}7gA9qrh7nhIzlks3t#Sv0)=EQ(Dhk_=XYfx&(6BEJ?IGap}Jl? zcN5SnSe8lbqg~6=@zI)D8!I!pw>?s>HOV<;$&oSi3y=W;ziV?Rku9ZTwUjnw*;?}} z`adSw)5qy)u*>_w9dPKn2~8Nc+n4Cj`Rs)U3l;bka5mi<_{950+~Q+lW_eA527j}{ zTdNhUg&!YF1mqliHHIWN6*uGjGz1nL8G!_A^&aF%{Hb$yG?mRa#+J`Tll=!Fe_T@eeKOK%LY9qQLZGxA0!b)(+~Y5nXezC&qYFH zKuGY1vu;W|9UskPlt79huUQJ)7N^cRzn4+*!DYhQCw-V@#5eU49F6!b0?>IMues^n zkNq>Srq!`o-n{dFt>O$ib+YE{63FZykPK@CL~L^4*#VNcX9Q0U{bhN7)-*VrCivCc zr5{ppG~}X{aO|0GtR!g@hUr*_89Apy9RE~fnd+0yGOJH_&b^HIN3W-Ga@fJ5^MXz5 zvu1N8jqqWlfVm@5=4u+fw~59B7}a14TR=^ah1`%?3!Cjx^Z0Am@t3e!bkb9l z+Y=C|E$HT{1Puc1YWe9E?0C(EtftyYs9c+vJRywv=I`k9N=jSEh;zX;xa zCQl_VKPMsR(0BoXh$c+N?~LDWxF5b{jfUwrw5c3>Ix<=_216V^Bjt2J9GSdRU#$dng&#;ebw7QB4g5v(psVym z^~Hxp$0WqRdIUK+Bsl1afA2t#nR(jyYjrEOvnBi;9j>8AnXBXwcdAyIHDrRIa6UIm zLN;6|6=vd&bgWdD%Fg%^u59#`7;IXP>fV*Tqp|R#5|3-#do*n&!oPYwOf__>eGvN- z9h@df{~rigxk;Oi&pqRcseTG#tu94*P@2$FMNt{mPJ>MY7PrXnP zeY(G)I={yG;4St-y=P~$AGW#)3U-ok z>)JKZx0Pd6`SgxKX(2cw-fwYHh=wX)AK#pyeFTBDqLBfT^N+G0D?=$I|@IK z17oT?B5f-lkYn)J)MrA!bwl-AqA2trRzu6A>@$32>qR>hP5_rz%_Q}BJWC7J3!0?59 zu22cay?SU@5GYna>%kGxlD;urW zHOVeIY5Q`+qr3`<!ZZqr)pjb8J~fY&3kJaIRG`-ll836H?Nz2J6kkR~;-Sio zkUc^So#uE<9@yQ#k7+$SXZb{Ce!&AtNAf z&J=;vNGZ8NkZBL;ia#w&?7!#uz`^MGg5+A3!qLPvCb2y1?B9E@GxDNKl~^mnEQ{;5*LF|(bNN&UCi|UYT7jZ@HIGU(Pw$&mPW~)M6WaA;CYKc}pVwTT zSo?%at2$vogFlJOLV|Hm)tB3eRTooOIjvTWxRqe-PDvhpHIF)uq#_kn*&Ewd=l+K5 zpji!BGY|<(G#v)Z6D}(28akY}HY-3E#qhCget%|?qP&II{@dHiWj4GH_Fdyi+B?lC zHl2~|WLL?@n1_g}_$>d~S1Y)L>CsV*j%SKpO&~tnd0kR`WWi_^;D2z*UVdq2)!5bI z*JXuL_MSc%tDZMMdOQC>e}7#a^-8y+km_6+GzVkA^89@w0Uy}*f%!dQ(k&i zUgQkWtgyI_oVM%m{;%3Y-AKJ0dP;(G_BEP_KTwQa1DCFodnOXZ60ZtQhL1!)@eudj zYkr+54ucImiGwgpl7)CN&Rk18{l|y*2jAN_F-3C0JqzOBl{l|pEBv^qd{OlWjh0P^ zGN2S$e>P$1pS=IphdV7Q`QK!$ymgky+a5QTw_KYUWGTAf)=pQ)=+J$Mu6r#V#ka3z zON@3!7h%UQPM@&EevfX)oP_byR~5N)GH1c)ua0YB%$bSK|dO78kXtSDrMV z_;7p0u#9l7uV9e&JyD~ar&!OEnahF|HAlb3?;edA+m)xS$24Zh%$2BK7ptvh+_t73 zicis4rEAWQI^r8D1pQ-1?F936IxjByrfDJcv@8D7_njc059r6LsQxM_TdBey(}`xy zyC(r@8jA0HqT6M;(w0l#ep~vP<66x1-+Ny+OVf=@46PF%T39Q4AxrjL>9c?;p)}UOsAdF_7c& z!SmX$tCy_%{iS(1MwN_4-SZy?HChk-{)S5Ds8O+udx1<->9CKR&GiwF)!}Dv3n?n_e#Z$#9fx|mY(ysR-TqH+m48m$wq(2`r+VYF zOUS-*@vg60yS5xUxp3dU?mso5*>?5fK8@ZV#VvK=*YeXV&m%=1cPYSQlay(Dm>Neb({B%dw;Q5kx3C*ETr0^svD)moEcQPkI>w2FZ zG+#XRNrUsD3lbVn7@e#JfA-$cRC@j30GCkg{Ff3Omjr&)G_B-160NaCOB4W7)MKb~0aSUyzV$%y3E1@@h=^tJ;P;T+Pv! zZpQ|kQa-+Sk4EJQxKCSF6}b9{c)9UQl(^q;v;N*^VHvs9L)4Z{UGUKbK}w9PDZH`g zJb5pEAx+@J$lxs7X*Fg0?hxG{`d|v4$QH#FlIEk|UdtD3z8t?-OtAvEW|<^v<_{p| zKPwROxEkh2+$+zIdxaDgvFuuPvXDWewL@aLQf!87Ov}%fowI^)aHlhdyx(UMGGIKM z>N25Vc}cw9Q5eo0pl7Vus;-sSZ)_;8;<}OW70S5wEFH^cHn&)a_ugIe&H)bjwiT*=NlNDEOJw5e|($MZ8bwMU$yH*T zd+EbHI+mZRO_PfT$%FwWMS|@o=Ct{;GzrdTQV@UswAVHmB#MsP^0=pd8n=YUPJfjaKYBZL(iAuTk}!gkQzD&hCD9t7AJC5UgeiN$W^( zB(Fw9$j34)6?w!Pz!jKzn=7LF<5}cH*|O)jqFLnjureJW@~@gyoBAzMnhG-g=k~^G zFW;zVN>KKd40nUrtWN3jDlZ%*TpJh`=tDE>Y{60ex$>D7e}STu#nstAb95t>Pw!rS z{^g<_?*d+F)GxbPkuXnP>{0oW2U~+;&v(Y%*zdUK7P{R<@xc|JH7SE6+gdon6|nRn z59tKIKNH=5Z|o5J?aC3N$ZQ%pbnw;g=lM$fQa_qqfG!*5Ylz8~_U7IT+KGGnY~?ND z0jsy<%fzGpaTJL;{c)7YXXwbNs^9CM#JYT1cBB8h+@8p;A|;dJj;lL9D7<7(J!DoM z5jXd>{YYPZd-S>OV_#%Lu7scJ2Yl9>T^g${>o)ofR~XHJ3$lfc zUbF7Vl|*Eip~}d_x%(0CHec(NdeHs7J_{D+yTSDBa$?5shX;!LdPYt_APYWw4M%}q zxyc=znj6M*Y1udEa{CZv)3)lTXhsIlK=9-@<4o|EkO!&aD*pVD-OO^9BEL0-&JJu^ z>NUrP_ET?hUhh9!4-;tq9>sFI@SV?Qu-J#b0_s$n^w=|2O9MD5zMI5mKt2Ow<0y9k zTc|6-h_^eMUwsAK|$>=U;iM56pA!^ z;SeL8?o08RZwi=2rdfgZ*%XH7Yg=gl@_(N~vtG=}bgeZo4N35cCs3x_f(xJJI($4A z7)zzmM%p_BUSU|kBs`EeydM(uo5`S+wYKuj;v|quK1?cg3Oqf~J5b#`Gc{x8mpb<{ zCj#bg0El4)A$u4=-Xip4M_JVo6>LErLsv;0Yyi8|us=Bfh0-ZY=>StL6Sg2Pm^Knv zkBIS`GBn%{fHBwf6?*0;ocqYcp_xC9eBPTqf4PvU3SC`Y^{94j;feAXq4?vXZEfC9 zOHU6p)idn`q;m70m9%#*E~gZsTJKdf zK`Vnlz6+hht|-FB z3h1ouqV+l`(yIy+wC-PZp8|@gX)u6_(V+~dic(N8ef4ynq;!A^_T-?T8KI+pC;{-y>GAd zaX=F)x157G=Ne%SK^A~0?w0%BukrqNXiH|HW!%cNiM@Cf61f|pj`YRvyTN=#8$p=x zQLcRpX}2@YdA1)c|6i+@c%c3KwCx`r z0z<}bG=o(?8bw({tppmcyno;cddBy2ztDb<_wX!EO{h+0=&?9yh&W&Gp-yrVzOGK1 znF?m$JRpdlGr{t2OFI$RR0loyQylcCWah*yR6tl}1E|kWanQaDg*_TT8to2J9RCVQ zOSY{DlM@a{53?TBN?QV%U{=ZwxXSjyIoG-89yarJ0i%g~x$~q4$GUq&f4@udWmZmR zRj4-WVbOea>xmEkTzT6-ktg21cD{pzIBj8AT8bSiwdMcbYVQLH%QUvE}Y@%+m*+Ftw4Bo&9^zegN~X@6>P3r zDAi<&M{>Hr3p(t}*{^7y$oq8`Czrw+u(%r+d66yWZrAiQ6_+gB+DfF>VXCT5M`X!gL^g6rLEqamk%z)x(pXFaDX|UfXh(!;=i#aZiB`Iw*{O zaCGWX_N3fHbjAn{H$ZJM zYxX3ds07K>3crBPYEVMI+_atEYf!>xM9W_g_}N0N%w%Y9D|?*zEWGvq{0VfYQz6iH z{CgDH`eWUNfw*~ZT3k97PITTlKpLs(q}PtGzx>P!^-dXd?2m5Xr@d8iG^GeJ%MzT> zSaKHhn9D^6vY66-?EUUKD}z5F_`;P#k|R(VT*bA+#{ zE@6M1e0&rSGK;jqcY8fsi0X{A$EKW7Q#WEAEWLM=7Q-mdwS?I;l`O>-2+aSz^KA{( zG~6w*1^qCwA8=?t&(VP7)P}TPpf{-dQXcz18wce;aX`iwRBu&n3ULU{IVh`>H-kj8 z<7-lS9}+a-(6m3Fb}Qw-t>Jn96y6UczAsI~IN7*<4q|_{w4WggGUz~R1Dad_!CuMp zr+r!8H;kNQhEbV>bW*D7$F37-U&rqOe9^o-NpotT_q6 zn@D|jfGXApdUz(3K)Ii>(dvVtEP*ERo__5}5*;!}{QtHw~go zbs!h%f-TtOqI%zegY-S_BN7&*)9Uhx^efPrEA~M5ksTasV~J|W6;_3tv6 zk}-6VMWZ(XHB~c|N-Euj^jkfPX_xmJa*Vz!FoG)rQQJhwxMyK^ykim%mV1Ek^V&d> z_VU1E4E}pdFo`8<1N~GmdscYh6aCn7JfL{3P!Chr@E|^*Bsaq$u71XW_6D+|R|`Mx zZ3Z9PsV9a#&xZP(I3<3L__(8yFoKDOj+%W2Y<;OzSKNp`aD&M9+R^-) z_Q||Y6iw9t0gL8*;UiM532Aobu89dk&jxp5V6(+XUZT8p;!|jp1*=|{FrE6a!}d8{qCKeM@023K1c9Ym z3aiQUOX!znm(Rh>i%hhPnqb-99wo852zd5UBAq&HhYW$}1RKd@v|9s&Gl->Emfr&M zN*|8*Q?Y88TX6)=1c{g^RGE^~D@>&V4azN}JrY>~lmQ*scxmX;>f`VuIJu-<(>80Q z?^RSDkXI$KjY)s9xeWjF=VM?P_7x~B6)&j55 z&t$$|T?uU;%YI-IBZoKjEU=>ZjN4RC`!Thj$n*S6tUa%|pr2lsP)fUjftITJg@=;# z%Hw@6he`@Xfe$2-cUFLi8ApG@{9zR}A2Nt6?6Taqc@jZm;P+c1kZk;u(0NM~?-aR(Mv1z0> ztw%Y@Yi{U1dsZz^V*9golNQ6W+rh#(vz*;^P3Fkln-xkRx->|HQ@_@h5t7NzNV4 zNsTuNg5+M~cxx4{Ax`1dWjel!s1Q4*jA9py>vfXpW|r}B;KS&EY1#Pri2Sy@(Nqg| zpYC@SjakX=lZaUQ_bS$Qww!3pu+(~?@>a^OiPIA@==k92JA77`EX3b|OV?*iTrHqL z9S)vee2`PQW66Rz$C*`n_hflD>w@ft?r&J2^>%v&EKgGVb$Tn7YNF$KKeD ze^3VbvE#L3>RWJ6SQi~POT+o*H;WVfI^7qPA~bP_MIs((UzpX8uwR3p&3wv~hm&>A z70TADzZWL^mWG|aYjhiTX8kGbq|u^{ zS&VEzVc}IscD=AEsZG}LWjMjL-cdy{iI4;ksv9DCIx2U{9=sS^bq!yO`` zeiRK)NY)PBR=ejD_zc$JZuJo}S19sws5t$q3Dos(t>GOhN8z6Z?6w3gt`np`nU5km z3sBNZ5Ka~il5vX!;30lk8Nu|(^Fod}aRJfRM{IM5QNJnne7|%TJCR(6h#al~m-Gm! z$F>w#dhL)<92rLw6#zcfC1L}qzPT2%=W&RvBOUHN)&0ckch zaS)ad+z1YT{t#&kH{~8t8iA1_cr<=v8@%qnM8Vb3jHoII6@U;PaoQ2tn?8JEmZo$- z%_8D=ot8>^-=1&U1o!*Uu&~d@U-gHNz=h|P%yk6{?OdKIWc&mzSO-75<}_cvgA4WJpBq9pQo)~Yl|j8L;D;EOpUh=vOlyOR)YJIU^k3v77m z5K9c=oskHGnkLB3ygSCS`ay`t(DS*(5CLR^!OQvo#<97KoLon~;V*N%6_CsH`>nKl z3`P2@v(p+%+~zoKPCaVH6tbfm>6JwBB}x%;O#(*X#+@KTYtdeK9)GbHrO7*UQvCRG zGP}IHEuF4cN;_#Nj=bK{;s+~0`lMo_V1^4M~m#U1PFV}-X`!?=U;nOmtlEh5+J-?@cFZ- zva(<`LY0zX@4G2s9a=rZjiaykdalADJW^FMkY;8o8NS|21{N7%oun@nO8L-HO1h8O zKxAx}bLHR4RQh~yEBaPnc)lYYtV7^89Ew2!)FVnSe2c3oekRC4n`!%vVitz_s^S;j zeSr4~B2YP`_SQhx5I8-Eokd*ke>h6H1%(f~)BMU|4DGvJb>$IZG z++xDx6J$?ARH(-q?gJ|$6-RbG&J%m&?Z{l{dntuUj0ib3mB`R#(a2!N6>dpj(-KNB z;hKYyT1=E>6JV%S7#Y%@R3J>16E$M}eh#*_V(yL>*=~m06Nx_T+zJR(9um?c>8&7T z%_O!L;95U8#X+y0%$z7MAE9D$Mmp38Ww6n>V0;k5tmd|xo58=wGQCrse7F|a%5mUE z%4dS;*PAt0>=)Wzre!;a{YlM3viBXZFG$8LYtg_|fV%FWvGu`vW8az0^xqO?659!L zjVpv&ID%!=h}da|?p+!@q@ETC-K4z`P`ywoANF~4M4kJu(QjD*$WC$4R|sye{0+Aa z7Ml^grN;Rjs}vv#ulQ@Z)(%_5WG#Slk0_n=Uz|lL z0x#YIQlc|HSFpwj+R|*uQEt)s8V}5h91Vm!*Tr7^1-W2q*uTr%fjJQpjps7faj)Sy zadgh6okX~z4#}B%p9tc`*;IOpMCm+8=>X8-9J`6ALac$PYL<#)`{kP?#vT7~Tffts zequ^5FZ>wS)KYv|+hwpeu9fypdD9U_qRTnbfHVY$R(_#}g}__LEE0@zv)rc;yfqE> z-*BA>qpyWq{5aDIi>sWSz)7IStxGfqdEqGO5wFaT?xH;7H;L2{}n08WriOaL}WXLuEol)z`7K_oDY?P(Y7hRJrny~xU`8MAl(j86-S z!VCnZdIGW|X(lAXhd`-V3^Q;Zff8~oUe)~F8`{^SXqXx9EAVAQs&=0A@e2*N0BD^i=^ZUC7KcglJ)-muma<089I+zI;fz9w@OP@L7WHZ(K&s{B zH4EaCN>Y#)85(3YklKZ4Xu@_qz*1JwHlW?BKsI#fcyR7#zsy_4tVWX4_|ulR;2%*3 zLOFAGFztoB>$Q}HM%!d!CPm9*aU7HLybkz$5;5SPq{klKmrACF7_mxh!p?i?h{fPI zwcA^S$&KcO$I;Za54I;HNZ|NdCT~gK2>WM?k)nVnh`qQDozf(XSqZg%Hr~4-VyA1G z2U<_WJ*SZfBZw4YPdH4$-w=^0rXXdZ_a2vfOD zl2^3-;_V;iz~B#MYfd&V$ol4ywNzXPF>=~rAyAlvvZ7`1KL;ypu6QbynGb~__y%fBJbEXRMI=?`QXJnns) zsoU9^<8!z9Xram(vVMP#{y6Rtzznv{8F&uoy%29QNxz8b2{z#V8%lZ@k+x3d|HeAq^XCZ2ryLBZ5r^FErd@y ze(v`1Z7_I-0*H=8Q1s0}>sijeZlgR`(bb*bi%)T^JK5qnVVeskLzA`oC}AjV(BK=& zr9~F+cdLC!a*v|5)o_-_M>4Mz#wxLL`n8}z9|ht0xGKLE!5M*wpV_^8sBo4uDx<+u z(x@mxw3!nTH0$y)-8kk@qNe%%=}iZh{FW{i`qRGL@v*@VyA+K;Gy?Rg9_=DCspeB!cezT4t%=m&-Xj(OtuzNKzs!xVM7|m_F-Gb%pILL4 ztI`lI%Dl@s4X}sI$ciRbOBn9Gb0r4GoBDDJ>pFb*SZs;p$Q#8^n#&M6Gl}w#X5XbCYzrbR&Q=8`knXe~DaD>71}w@7%EW^ra)fr^0A=lf6@_yFnHAnOcMuvfBN zzJ1~or3C|u!o$6+kN)_SfYxH}UpOxT`mpXFo%9IM6sp1CPtFur!PubNXy^MCiDf;K zyylmKpb2LnVb%w1cJq&P1LD60!J8TUHy5`M(J|q{YE66ugLdfxsR^;?(dNeuV32GVG0bU(&#cCG1?nsMh-cY+~gU}_j$>iD2_%4vM z+M>|v*8gwqUWb-K(=a5XcoP6c;7ouvmvB*)-UN4)^mc$72q^{8#pK`lfG>zxp229V z-FWqxCINe)NrbnKtP-$1mKL#HTe;fw+JG}B+8y}@vMuJF)F7w*j7#d;+GV&;du+nuo&!#GXScLnPLU0 zsWbmp6#5pZ`O~qEo4~ETHE@*vMGcuvUC4>ANKZw)`+_$M{kt*-{w8Ow@6c4X1tFB= zFo*NqGOz$e=Ec^_ReN>PODs1GgYEC93(tV*k%E7LI+ zM_Hj#%K9jm7?N2LtA-t*k|rbWI2W(Epob@h;{Vm=n-~s|sYlNpA+9gecuRYQ?1wpG;;vN0+kWFP zM5*{7h)E->Q;ZtXRE-$;M8AiI3rzhvH@}dves%@OxdDbT{8dmNe6_U9JMuWj@SU{0 z3p8&FjS&tu*hSpTJn4-9OYZz1_gKoVFA5F5!Tq`|1QyM32@lCg(o7K*6Ge@d+e7TD8I>p*E#oxeU;q`loQoX;~MYs`v+FL<9;U-uTAu;;uBxdZXxeOEoJyl6oXku zSqXb#TYM`f(H|G%k+TeVkOCI;W)5q8`lW(KAl~o^WQsJL+wYk~FdKOz=@cL_+1R~> zIA7uh<Bjv8W3>+8W(U}}NZFA>?WRtEo3ehuk?5|Gd6t9JtEKi=RlAk( z7?9|JY*44#+Z49oyYHqZ@zNtb-gfc-oBEx8q2%Ox$s$|zPpcWna?~(Kc<|4S!D`ng= zY+LfA6U{Ll&I}Evdl5TgfjlZ0|A(1?6swhq5AfjPTo@LxJ@W#?gzyN6D+4g260n7g zHg^3Bd$7-tJ+M_>?bwTmz2(hgS5uKC72>$z@VwUjhNysu@u}nB9Vz=L7B|7R_V~NB zAWpdz8*J{4IPwA-mJ_78(1-TdnZ+7`a^O9%dp=_12DDF@Qzye!>A@$>ZMGjU)S`eK z;)tAt0zp){UFOwv`jjUq$#n=i=5Y5>J5x@-)mc{ymoV{*#~g*0i}<>dk{kvpbvVC| zky?`UX+UP^LsugMV6RN`YkFwSUQ#$z1(i985meffq0|fUYbw*;-n)r(p3W`aWdU=| z17|kVUM0Jk8R;R&aio#nNapKwKO4oFQu!CI_myEmJxUg0>qQnSFu~B7vmF!O{`?7y z#D0B>WIqWt>2l`tO&vf=vk=Z}NXkumX9$JlT)TgYgI0%9IyEUB4<4qZ0c@t|R{hR)Py#bdv4j)_Jh=gB9BM7!w>p@I<5 z@f(M!1I?rpM)vXlfXFw{nn9%=oDU;Og6+El<4;DIAD}XuSzI54!uK$eux5(@n5pqK zE712#Jx}3K zq;0a&wSBPx=6JjS)X?X<9Qj&0aSS#3nurFP~U?Nzd@I9udZ!$fV#`OIRlil9di zv=!Ou-=6(o8$Sc^)u#k|*8%C~e-ATIe{TGJ6zIe}onAwK|DtQ4vv^INC4*Lp-4W*f zYN7)ZNhW#`GjGy2CieybvEc6|DMvYmo|^M>CXPB(fh)H@-VBSFmD^6F)YA_U zN&m%CD4oj8i6EZTFG5+ykvH$W-2Bx5(Xxn>7^#x}g-j@B$0)}L#|Iaa3pVV23+?lY z`Xtc=-e61Mp0e>VIPgBkhIvXA%CAycAK4j7$K7tN1K3K;$BEB4LEi|b`3&RiXZ$DN zHIju`qzBW9=5)_zU7?4YTCc&)_eoMHjw9`y-Sk*SK8yxBr`b+h^IRmfOPHbC;G`ymPL8~fUsyz9yV9S-aqcL<*GTtd+sDY>V7)`T zubp-jdGXIL!HTmcAQ2fkFh6I#dUz*~C$r&2MS&)ljsRy!fAq7Seu=r#^jYVlq7OZl zO4C<_Su#w8onOpfUq|3TV&TbRu~+zTR+9TQML~FyUsQZPz)i-sKR)$xAlJLo;mB0}`srd}3WfD(=e-Sh?3#*PI3wFSi!Q?#56@ zhVtO?6PEu0-r_*TUd|rBCHhDoIp_q`!}Ub12jLX8xbK93xmWiNK-xx0S`0+ z{fSSAj?{$WTvtCjNow!}jaV{EB4y9+#`AK~XPq?HZryaAczd_kqASlx?=;932_Ip& z5B8fsCxHl`5UUdCHvf|>@QeL_qnc!@X(FtjjTQU!wzQO`V81hc$@+>S*x`H0G$}ti zAyKhuhIbNTNxuq;qY>B;<%-MWwp0}EmZR-GikTlCIn4Iqr9@i=q(q=7B5nGs0deph zk`>vrYE85bM6u$!3M1NYP`r~keMm!?%oymy3G^5SG5RQhlYoEKU^wOPBb*+Y9fiEV zp){&PPH2zm6QQ;^KQ0t(aB;vWSHn(yDR zj6hE5NJ5+-`pJ8_#k33JjZiZELAZW6frkF6CU%O?{fSW%tqFFXg`)uni7f=d10KY2 zfIuLAi?~rL)0Yc|44~^#P2g~HdJw!1)qg*kxQC*DV8}S&9Va-ngeM@H5PkOz@$cr| z;jFf=o_2R-L+=H;;X5@sU7D_J@sL$HWPYvj*_Um-V#gSlE#qc8F1G#@yY@T%Ia|+3 zgQ=q!siCylTRNKaAm=i+tCywMaEcxKaFNrY_weA@)Wnn>$Dw9*mQ#BQ$_z4da&aQR z=t9>pomzh)rzh?Cxw-eNIYWFoQs)*S%?l%3jtC@=l}@vFuT{A|^Ph^U`0iPpupl%u z@JIzU)Z98bQzy3NrNgQNLW#gA)kg*e|Gf0NfgYs;6k4_22(QhT+)P$YqZm;S9b89$ z!AZ)o57M92c4NPElNS*3+}rQ@v=7r{S!B8qM1Y>pz%K=QJ<#t|+Z}e;F8hC8(+Bj} za?=$Qi00URMzJkho%`yNPy|{IE@;_A`?Pza5~~aAKpRSj`L!9UHixXY0ug3Fq4e|L z43FQ7unWJ6vamlrIjh0yvWG71_rxuKt$FA`d`6wo@h}4rRR-GeRh!h;%S#b8?%RM; zR|=9tAb2p_ad;VyfY4=*lKrh6xG5Qogo;y-2OHjE@}7}pqnN_9$nSf3eXS+eeAA6j z+k6w**+*4Q#sC8C(U{+LZyRx^?$KEIW(j)`?OtTw$9-oFP8b~YyBbFOORcc*#iqYH zR!>_DJmRHY@4d(POG7yGy`Nl0dtn8;WOMM8EbS$6iJnY;YY(LtC2o?W>IVLPmH4fv zjPkTAt6nvWllISv-D(o+;+`SH$cUzb5*rqjY{v1`J!L@uq;+$}u)R{*^YpNZ(yddn?`0QUrTu<5 zV&dz+FSv^rAn*CGe_Q)MeW(`ICrpXg+55>5 z`rDh0T(A*7i3X<`fO2axg~c3GPKw<|`fjQ2%e1T%KqJrZm>Bj|4f|LW{cGZeK88Ol z8WzyKhyR2NYx$rx3UEo!U!!lwn`v*a$+BV1c}#WLdKhHSGyGTL!8!0sfsm7zHG3%ZE6MuXwS*Og2G>-@LufYf_nJMhV$+;sd(CH8yinms;48??3{dt(((jizumy2xaCj%MRcZzrjNe4${m3P7|vYhtZXIR zxL8H~NaLDBA)6iAEM!RVqdZg5r_$9Z$Wbo6LqQ(KgW0hbS%r)&STa`Mx`AQXfL>uU z@Mz7treYM$Kf9;6uHV~|#eF@zWJF$5i@Yy$<04C8!8Mu1d{@lSr1v0C#wPM~&0DAv zd%CAo>5nr0QQ@|qRa%v0IN69@?*Y=kusOd>r&#B(ZDtZ=1ofvXC03o2hRExFp%i`T zhzNK)po}>I;-G55aJsgzIq_+86AxurKzn90F^;n5%J@{ufxm-xEY>BA=arqjm}oB* zKaVeb3}+th)L*f~?@VDR{f@s_U`k)x;gt`>H}5#QkM=dx8Bz-WM2@v868G;(ek0#= z{g^Z~gCPN{cY9vHqf2{jW*}uwb<(_TdeyYM5Eq)4t3Mhz8JF`D$BsPR`;=APcE;1?>d9m zHx;f6`1N0?I)yh^cTDyD6nkw~2|D->e@s9Q#&x!J_0-UXAH%?o-;><14^8Xa?^?Ah zs(J}?e&*R!y$G~eb}W?G0%2jtC1y;z03!kLF_DyAa9z9_L@>*=9@GX|ko?_iov)0+ zNb7&{lnDpGeN6l?4Dtv;L?*(aXvO`HQX%4*<|67+kw%;<0R$C~S8`1||NNjz^~d9T zSF+j`AqT%fDr|j_msf_|U8mBH9^YGx9Q!VQwchN8x@PXtF`Gn(Rg?R|J7Z>7`Lc-K zgt?Yu{hjGwpR4LuZ!uG@ruuX5kj#aTeHx&O^@lCvLrAx$WWK7(q4ZF_#M2zj*gGP?bA8v-gnm_N=|=IM?e!KP-v7=8KYiMN(u>I0GX-PNdc& zpo4z)SfXMz&m$1gU0~WhtNwWU*{mw%4~ta8Hy-69FwcjXNbU|}#D_dyVEP|NYHu<= z30~n=mo=8YBb|7#WxyvRMY8ISTE^F)yA_$7Rw7)#{IE?BBUjeSii|Gm@pW5-3TLk+ zr-=;_cmWa!23^EC(JQT7aiuv?dYhZBJ!B)r1Ga1*#%&ZsA{@mHs%t$;`Us39l1$$6 z28d5OK+e0j3R2??4BGWuUAhl=jcY~uXTDmx&kS^}F7{Esij4N@0>`zMORuOg3aq$g za~3Q}P47=2MW}gkXiR*=>eyGXl{NeMoRx9=Wf$NQR5-TzDdVCTlCvT@P2QA!j76mr#db7KT)^(6r%9~uWL8~tifiST$C7u-5I{|lYm<2 zQnt$nc7EGSC^q*)mXo;ZCgRIZJR-Sx-$7+j567Gv#&}GbNpF1xE0>h)_-?jqX`ZPG z_m(vr`es2HeCG-U7jNN3&e^H0OSG5veqNX2R06(loGow%C{+^3AoZXX7_NUFZIC9f zHfPCg(?jb${Ik2mHzeoWRa>`*RbUZnqnm^iEc7sZ_7%|Cs-`Yf$tk#R0f$7T*F;BH!Y}9sJ`g4+r1aa8>_{5*d-1IOu>rR$QuT=G6R4 zoNJvO7pdw`K^1R#$S=+>Q#kvWf1FC7h*AG(V|VnwD!*F`@adrR?JJm`wC9?Avg0_T zBfN{>qm&D>#Y#>LJ{K2yNrmEj+f2Q3>_l3S>1=L~w3V)6vU*72(hGidZR(LhS7iS= zrnGgIx|mqzt8GxCqHNhKJ7*ZV$enSB7!63>PG74Y!8)9M$x0-xq9};CD>F z{O#=LsHdX1o!)KMM}C!#hqz+`3FFPd0)>PBY}q_pT=d!Kr(oqH(%Tcwk7j;m-jEbH z&TX1LKtvZCdn}{?ZzhwuKxM>F=m!;>@y_-2^RB8|tso4RJ}eLAjvz0%>Ito3#1N_@ z&PevV`W50kFE7(I+)PA#=_Jo1qwj_{2gHrn?+6nwk167x^=o|2*f%aIO97tjBU<;P zB0S7=O)ySK$mO|WEAgIi`|$KZ)fTH)j>%g$llj}nkyq_!3kJCDJo3nEa)bF{v?a&& z(WlfjJ$LNw_3rF1U)a{JD}4SB|2pn@XT@0G&dI-S)ks1d84o0seyfe~hHsYzo1U%0 z9iO!qgZP#1wVBP{1bFy)A4a^WKfio*+9%69gk$!moBo7Vt`#BY&U%A6LndP#)Ov=@ zmwhGuc3p}~H2JYvV9vuAv5Sc%vN>dgR=p-4?Ot6jCm}d<)5GK}gY5%oW~n^P<4fIF zQFZNWcAe+R$zfhrC-}OkPtI30Vf(RmUh2CorEzd6?^mx!|R@4jPIP zo)?`~Yj)u15-;aVAH&8uP=6VQAu4SC8+|suS%@`%;zNzzu>exc_DBnC*yCJ@n;qoY zilLJgdlfd828-Kfaqq-~y&}%~HuJ+Per}K6O07@l-zo|)y97x3Ris!pE5H2`s@*6UJ5*aRB{8{w=o83jDO(vh+2$fkcJ1Bw*uZykBk1FmeV9U0 ziff2s|G4&x$}QEs)9r|V`{uKmzO-uy`97aA^NK9IhyA_UfBvckOI6)HsLS)U;1lp5XJX z=CCnSaPGoqgc=Xe5mnEL(odyn(^P|2BS zDd$sAs_W^nYG2lKV@@@>Exdu+xd~(MnBT7p)#Gafrc-Hl_mYiqhgQwzjwyR?yj)G^y7d3k-kC>3{fF_skfq3;eIH{lWY1c} zj2Sy2MT0~nSt45~*<&nOvcym#JE>$T>nOV{2@w)fk~Q4t`}^ab``5kqkNf96=XW}% z-#J-k=KJ|P&-;13-_Kjs)L(Duo>~eV*o>Bx{;b_aKC_rFZL}^%c{}g($yB z`A2aNw3!sqqN4b{^a4Ta7S0G3uhN=vxH1Nfclc5gXScsQ_MLbPhoWm^#C|IrUaQM+ zDP`rYI7~erEqQq88n8i^!h{rwS^kF4CDDxabo$g3hA1#4erLPuVA6trgrQQ~giwK- zWD^IGFL>oI3SZ(12B#pvem!qeB$+1V{uPQa$Mke3RV9_itpH-t3p&Fp9>+qeU7L#5 zOv@7~u3%C02$4>XrBd6X3+H}T_wvZfv1A4u?&b}&P^rGma8YN%Pf&wHJRZ&dg+1hA zsVNW`MTWbh_mr2j*30}tVTq^ZZ&)C!x?$k#1_oc}MC8a<|S-BRH zZsF@;Vo|AhWjL}|;nK@xXY-uzY`ZJ$5%lUEDiI-Vz>=qL^3jHy?BDmhzu$5E%yMGW z1HTA4hFP9O*%q?wj%feo`&-#v9nd5xBJNx8iakSi)*0_5($-@$@5m)MglddawV$jx zS0hxzvm9SNN{GcXP5k>jwV$@}eiQ`uL4Gb?{WFy?kYDhiS%^kS?TG(wAVt2SIA>;w zqcZ!PbeJ+CSf1^b&}q!g=`flicUOL}0(VxYJTt}D9X4#7y~=B2iYPIeiBwrXSvs3x zRV-1dv*4$+4IOxvifKw}Vlo~MS4H{fFx=^?DJtcCreR!UCg;z*>T;I2fgWe3rcmE{ zGvHzpB58F+_Pc|yj+sZy(w(A=tC`caGk=32W&CNhA)0O`Ae?`l191x~*l0@>6Pm)j z2d!kUV9rvNC?%-I<+=?4zU9`yX`DsSM`Ge4^fMyEk?|FuAJZ_5PT-tirD zr2xn7%_9`tr6*_>V(%TpFr0=GnnI4N>tp8O!<2_MUO1E=yD>0Sve3eC2T^iB{3%&R zGI%l)eU5%__D-frrs@CvG*TlX@%uWQF&Yd^h!mnQnioeYiaPmreZXQ??WU0!D`~vt zFDK<8G+n_u1>c$*zqBZ01oAsURizou-xgwczwkkl3GEaysco;Ok!X*c`DvIWCGmvz=WReJj8sn z7Me0&Y{O${(~5&+f@vu-wxDb_dk5Q7J0iN;@1L+0f7`P=1()rRadElvxnJS zn@(1zO`_rO6lXW-~%8 zQZLglGc4;uGG&}GRlN*+fmV))myLBOJporZs`b?Rav8k#UT@9Ixl4P5pV``TNyz6D zh!MjMre#>Om30lWoLn=Nh^W0mW}drYUMnj2@J372^<9vajaQD7Y4-XPTHT~~#MJc# z*OC1%86!Ab